Changeset 56248 in webkit


Ignore:
Timestamp:
Mar 19, 2010 11:23:57 AM (14 years ago)
Author:
eric@webkit.org
Message:

2010-03-19 Eric Seidel <eric@webkit.org>

Reviewed by Adam Barth.

Help sheriff-bot avoid warning about flaky tests (and add more unit testing)
https://bugs.webkit.org/show_bug.cgi?id=36354

  • Scripts/webkitpy/buildbot.py:
    • Make Build creation easier to Mock and test
  • Scripts/webkitpy/buildbot_unittest.py:
    • Test finding green to red transitions and suspect revisions
  • Scripts/webkitpy/commands/queries.py:
    • Make what-broke note when builders have only failed once.
Location:
trunk/WebKitTools
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • trunk/WebKitTools/ChangeLog

    r56245 r56248  
     12010-03-19  Eric Seidel  <eric@webkit.org>
     2
     3        Reviewed by Adam Barth.
     4
     5        Help sheriff-bot avoid warning about flaky tests (and add more unit testing)
     6        https://bugs.webkit.org/show_bug.cgi?id=36354
     7
     8        * Scripts/webkitpy/buildbot.py:
     9         - Make Build creation easier to Mock and test
     10        * Scripts/webkitpy/buildbot_unittest.py:
     11         - Test finding green to red transitions and suspect revisions
     12        * Scripts/webkitpy/commands/queries.py:
     13         - Make what-broke note when builders have only failed once.
     14
    1152010-03-19  Dirk Pranke  <dpranke@chromium.org>
    216
  • trunk/WebKitTools/Scripts/webkitpy/buildbot.py

    r56231 r56248  
    5454        return "http://%s/builders/%s" % (self._buildbot.buildbot_host, urllib.quote(self._name))
    5555
     56    # This provides a single place to mock
     57    def _fetch_build(self, build_number):
     58        build_dictionary = self._buildbot._fetch_xmlrpc_build_dictionary(self, build_number)
     59        if not build_dictionary:
     60            return None
     61        return Build(self,
     62            build_number=int(build_dictionary['number']),
     63            revision=int(build_dictionary['revision']),
     64            is_green=(build_dictionary['results'] == 0) # Undocumented, buildbot XMLRPC, 0 seems to mean "pass"
     65        )
     66
    5667    def build(self, build_number):
    5768        cached_build = self._builds_cache.get(build_number)
     
    5970            return cached_build
    6071
    61         build_dictionary = self._buildbot._fetch_xmlrpc_build_dictionary(self, build_number)
    62         if not build_dictionary:
    63             return None
    64         build = Build(build_dictionary, self)
     72        build = self._fetch_build(build_number)
    6573        self._builds_cache[build_number] = build
    6674        return build
     
    8896        return (green_build, red_build)
    8997
    90     def suspect_revisions_for_green_to_red_transition(self, red_build_number, look_back_limit=30):
     98    def suspect_revisions_for_green_to_red_transition(self, red_build_number, look_back_limit=30, avoid_flakey_tests=True):
    9199        (last_green_build, first_red_build) = self.find_green_to_red_transition(red_build_number, look_back_limit)
    92100        if not last_green_build:
    93101            return [] # We ran off the limit of our search
     102        # if avoid_flakey_tests, require at least 2 red builds before we suspect a green to red transition.
     103        if avoid_flakey_tests and first_red_build == self.build(red_build_number):
     104            return []
    94105        suspect_revisions = range(first_red_build.revision(), last_green_build.revision(), -1)
    95106        suspect_revisions.reverse()
     
    98109
    99110class Build(object):
    100     def __init__(self, build_dictionary, builder):
     111    def __init__(self, builder, build_number, revision, is_green):
    101112        self._builder = builder
    102         # FIXME: Knowledge of the XMLRPC specifics should probably not go here.
    103         self._revision = int(build_dictionary['revision'])
    104         self._number = int(build_dictionary['number'])
    105         self._is_green = (build_dictionary['results'] == 0) # Undocumented, buildbot XMLRPC
     113        self._number = build_number
     114        self._revision = revision
     115        self._is_green = is_green
    106116
    107117    @staticmethod
  • trunk/WebKitTools/Scripts/webkitpy/buildbot_unittest.py

    r56045 r56248  
    2929import unittest
    3030
    31 from webkitpy.buildbot import BuildBot
     31from webkitpy.buildbot import BuildBot, Builder, Build
    3232
    3333from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup
    3434
     35
     36class BuilderTest(unittest.TestCase):
     37
     38    def setUp(self):
     39        self.buildbot = BuildBot()
     40        self.builder = Builder("Test Builder", self.buildbot)
     41        def _mock_fetch_build(build_number):
     42            return Build(
     43                builder=self.builder,
     44                build_number=build_number,
     45                revision=build_number + 1000,
     46                is_green=build_number < 4
     47            )
     48        self.builder._fetch_build = _mock_fetch_build
     49
     50    def test_find_green_to_red_transition(self):
     51        (green_build, red_build) = self.builder.find_green_to_red_transition(10)
     52        self.assertEqual(green_build.revision(), 1003)
     53        self.assertEqual(red_build.revision(), 1004)
     54
     55        (green_build, red_build) = self.builder.find_green_to_red_transition(10, look_back_limit=2)
     56        self.assertEqual(green_build, None)
     57        self.assertEqual(red_build.revision(), 1008)
     58
     59    def test_suspect_revisions_for_green_to_red_transition(self):
     60        self.assertEqual(self.builder.suspect_revisions_for_green_to_red_transition(10), [1004])
     61        self.assertEqual(self.builder.suspect_revisions_for_green_to_red_transition(10, look_back_limit=2), [])
     62        # Flakey test avoidance requires at least 2 red builds:
     63        self.assertEqual(self.builder.suspect_revisions_for_green_to_red_transition(4), [])
     64        self.assertEqual(self.builder.suspect_revisions_for_green_to_red_transition(4, avoid_flakey_tests=False), [1004])
     65        # Green builder:
     66        self.assertEqual(self.builder.suspect_revisions_for_green_to_red_transition(3), [])
     67
     68    def test_build_caching(self):
     69        self.assertEqual(self.builder.build(10), self.builder.build(10))
    3570
    3671class BuildBotTest(unittest.TestCase):
  • trunk/WebKitTools/Scripts/webkitpy/commands/queries.py

    r56231 r56248  
    128128    # FIXME: This is slightly different from Builder.suspect_revisions_for_green_to_red_transition
    129129    # due to needing to detect the "hit the limit" case an print a special message.
    130     def _print_blame_information_for_builder(self, builder_status, name_width):
     130    def _print_blame_information_for_builder(self, builder_status, name_width, avoid_flakey_tests=True):
    131131        builder = self.tool.buildbot.builder_with_name(builder_status["name"])
    132132        (last_green_build, first_red_build) = builder.find_green_to_red_transition(builder_status["build_number"])
     
    137137        suspect_revisions = range(first_red_build.revision(), last_green_build.revision(), -1)
    138138        suspect_revisions.reverse()
    139         self._print_builder_line(builder.name(), name_width, "FAIL (blame-list: %s)" % suspect_revisions)
     139        first_failure_message = ""
     140        if (first_red_build == builder.build(builder_status["build_number"])):
     141            first_failure_message = " FIRST FAILURE, possibly a flaky test"
     142        self._print_builder_line(builder.name(), name_width, "FAIL (blame-list: %s%s)" % (suspect_revisions, first_failure_message))
    140143        for revision in suspect_revisions:
    141144            commit_info = CommitInfo.commit_info_for_revision(self.tool.scm(), revision)
Note: See TracChangeset for help on using the changeset viewer.