Changeset 76160 in webkit


Ignore:
Timestamp:
Jan 19, 2011 2:12:20 PM (13 years ago)
Author:
dpranke@chromium.org
Message:

2011-01-19 Dirk Pranke <​dpranke@chromium.org>

Reviewed by Ojan Vafai.

Change webkitpy/layout_tests/port/test.py to use the
in-memory filesystem for cleaner unit testing. This
change allows us to kill a lot of code that was
specific to the test port, at the cost of being a
little less clear about how things would work if
you didn't have a filesystem.

​https://bugs.webkit.org/show_bug.cgi?id=52605

  • Scripts/webkitpy/layout_tests/port/test.py:
Location:
trunk/Tools
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/Tools/ChangeLog

    r76158 r76160  
     12011-01-19  Dirk Pranke  <dpranke@chromium.org>
     2
     3        Reviewed by Ojan Vafai.
     4
     5        Change webkitpy/layout_tests/port/test.py to use the
     6        in-memory filesystem for cleaner unit testing. This
     7        change allows us to kill a lot of code that was
     8        specific to the test port, at the cost of being a
     9        little less clear about how things would work if
     10        you didn't have a filesystem.
     11
     12        https://bugs.webkit.org/show_bug.cgi?id=52605
     13
     14        * Scripts/webkitpy/layout_tests/port/test.py:
     15
    1162011-01-19  Dirk Pranke  <dpranke@chromium.org>
    217

  • trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py

    r76045 r76160  
    3131from __future__ import with_statement
    3232
    33 import codecs
    34 import fnmatch
    35 import os
    36 import sys
    3733import time
     34
     35from webkitpy.common.system import filesystem_mock
    3836
    3937from webkitpy.layout_tests.layout_package import test_output
    … …  
    6563# what we want to claim are the expected results.
    6664class TestList:
    67     def __init__(self, port):
    68         self.port = port
     65    def __init__(self):
    6966        self.tests = {}
    7067
    … …  
    8582
    8683
    87 class TestPort(base.Port):
    88     """Test implementation of the Port interface."""
    89 
    90     def __init__(self, **kwargs):
    91         base.Port.__init__(self, **kwargs)
    92         tests = TestList(self)
    93         tests.add('failures/expected/checksum.html',
    94                   actual_checksum='checksum_fail-checksum')
    95         tests.add('failures/expected/crash.html', crash=True)
    96         tests.add('failures/expected/exception.html', exception=True)
    97         tests.add('failures/expected/timeout.html', timeout=True)
    98         tests.add('failures/expected/hang.html', hang=True)
    99         tests.add('failures/expected/missing_text.html',
    100                   expected_text=None)
    101         tests.add('failures/expected/image.html',
    102                   actual_image='image_fail-png',
    103                   expected_image='image-png')
    104         tests.add('failures/expected/image_checksum.html',
    105                   actual_checksum='image_checksum_fail-checksum',
    106                   actual_image='image_checksum_fail-png')
    107         tests.add('failures/expected/keyboard.html',
    108                   keyboard=True)
    109         tests.add('failures/expected/missing_check.html',
    110                   expected_checksum=None)
    111         tests.add('failures/expected/missing_image.html',
    112                   expected_image=None)
    113         tests.add('failures/expected/missing_text.html',
    114                   expected_text=None)
    115         tests.add('failures/expected/newlines_leading.html',
    116                   expected_text="\nfoo\n",
    117                   actual_text="foo\n")
    118         tests.add('failures/expected/newlines_trailing.html',
    119                   expected_text="foo\n\n",
    120                   actual_text="foo\n")
    121         tests.add('failures/expected/newlines_with_excess_CR.html',
    122                   expected_text="foo\r\r\r\n",
    123                   actual_text="foo\n")
    124         tests.add('failures/expected/text.html',
    125                   actual_text='text_fail-png')
    126         tests.add('failures/unexpected/crash.html', crash=True)
    127         tests.add('failures/unexpected/text-image-checksum.html',
    128                   actual_text='text-image-checksum_fail-txt',
    129                   actual_checksum='text-image-checksum_fail-checksum')
    130         tests.add('failures/unexpected/timeout.html', timeout=True)
    131         tests.add('http/tests/passes/text.html')
    132         tests.add('http/tests/ssl/text.html')
    133         tests.add('passes/error.html', error='stuff going to stderr')
    134         tests.add('passes/image.html')
    135         tests.add('passes/platform_image.html')
    136         # Text output files contain "\r\n" on Windows.  This may be
    137         # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling.
    138         tests.add('passes/text.html',
    139                   expected_text='\nfoo\n\n',
    140                   actual_text='\nfoo\r\n\r\r\n')
    141         tests.add('websocket/tests/passes/text.html')
    142         self._tests = tests
    143 
    144     def baseline_path(self):
    145         return os.path.join(self.layout_tests_dir(), 'platform',
    146                             self.name() + self.version())
    147 
    148     def baseline_search_path(self):
    149         return [self.baseline_path()]
    150 
    151     def check_build(self, needs_http):
    152         return True
    153 
    154     def diff_image(self, expected_contents, actual_contents,
    155                    diff_filename=None):
    156         diffed = actual_contents != expected_contents
    157         if diffed and diff_filename:
    158             with codecs.open(diff_filename, "w", "utf-8") as diff_fh:
    159                 diff_fh.write("< %s\n---\n> %s\n" %
    160                               (expected_contents, actual_contents))
    161         return diffed
    162 
    163     def expected_checksum(self, test):
    164         test = self.relative_test_filename(test)
    165         return self._tests[test].expected_checksum
    166 
    167     def expected_image(self, test):
    168         test = self.relative_test_filename(test)
    169         return self._tests[test].expected_image
    170 
    171     def expected_text(self, test):
    172         test = self.relative_test_filename(test)
    173         text = self._tests[test].expected_text
    174         if not text:
    175             text = ''
    176         return text
    177 
    178     def tests(self, paths):
    179         # Test the idea of port-specific overrides for test lists. Also
    180         # keep in memory to speed up the test harness.
    181         if not paths:
    182             paths = ['*']
    183 
    184         matched_tests = []
    185         for p in paths:
    186             if self.path_isdir(p):
    187                 matched_tests.extend(fnmatch.filter(self._tests.keys(), p + '*'))
    188             else:
    189                 matched_tests.extend(fnmatch.filter(self._tests.keys(), p))
    190         layout_tests_dir = self.layout_tests_dir()
    191         return set([os.path.join(layout_tests_dir, p) for p in matched_tests])
    192 
    193     def path_exists(self, path):
    194         # used by test_expectations.py and printing.py
    195         rpath = self.relative_test_filename(path)
    196         if rpath in self._tests:
    197             return True
    198         if self.path_isdir(rpath):
    199             return True
    200         if rpath.endswith('-expected.txt'):
    201             test = rpath.replace('-expected.txt', '.html')
    202             return (test in self._tests and
    203                     self._tests[test].expected_text)
    204         if rpath.endswith('-expected.checksum'):
    205             test = rpath.replace('-expected.checksum', '.html')
    206             return (test in self._tests and
    207                     self._tests[test].expected_checksum)
    208         if rpath.endswith('-expected.png'):
    209             test = rpath.replace('-expected.png', '.html')
    210             return (test in self._tests and
    211                     self._tests[test].expected_image)
    212         return False
    213 
    214     def layout_tests_dir(self):
    215         return self.path_from_webkit_base('Tools', 'Scripts',
    216                                           'webkitpy', 'layout_tests', 'data', 'LayoutTests')
    217 
    218     def path_isdir(self, path):
    219         # Used by test_expectations.py
    220         #
    221         # We assume that a path is a directory if we have any tests
    222         # that whose prefix matches the path plus a directory modifier
    223         # and not a file extension.
    224         if path[-1] != '/':
    225             path += '/'
    226 
    227         # FIXME: Directories can have a dot in the name. We should
    228         # probably maintain a white list of known cases like CSS2.1
    229         # and check it here in the future.
    230         if path.find('.') != -1:
    231             # extension separator found, assume this is a file
    232             return False
    233 
    234         # strip out layout tests directory path if found. The tests
    235         # keys are relative to it.
    236         tests_dir = self.layout_tests_dir()
    237         if path.startswith(tests_dir):
    238             path = path[len(tests_dir) + 1:]
    239 
    240         return any([t.startswith(path) for t in self._tests.keys()])
    241 
    242     def test_dirs(self):
    243         return ['passes', 'failures']
    244 
    245     def name(self):
    246         return self._name
    247 
    248     def _path_to_wdiff(self):
    249         return None
    250 
    251     def results_directory(self):
    252         return '/tmp/' + self.get_option('results_directory')
    253 
    254     def setup_test_run(self):
    255         pass
    256 
    257     def create_driver(self, worker_number):
    258         return TestDriver(self, worker_number)
    259 
    260     def start_http_server(self):
    261         pass
    262 
    263     def start_websocket_server(self):
    264         pass
    265 
    266     def stop_http_server(self):
    267         pass
    268 
    269     def stop_websocket_server(self):
    270         pass
    271 
    272     def test_expectations(self):
    273         """Returns the test expectations for this port.
    274 
    275         Basically this string should contain the equivalent of a
    276         test_expectations file. See test_expectations.py for more details."""
    277         return """
     84def unit_test_list():
     85    tests = TestList()
     86    tests.add('failures/expected/checksum.html',
     87                actual_checksum='checksum_fail-checksum')
     88    tests.add('failures/expected/crash.html', crash=True)
     89    tests.add('failures/expected/exception.html', exception=True)
     90    tests.add('failures/expected/timeout.html', timeout=True)
     91    tests.add('failures/expected/hang.html', hang=True)
     92    tests.add('failures/expected/missing_text.html',
     93                expected_text=None)
     94    tests.add('failures/expected/image.html',
     95                actual_image='image_fail-png',
     96                expected_image='image-png')
     97    tests.add('failures/expected/image_checksum.html',
     98                actual_checksum='image_checksum_fail-checksum',
     99                actual_image='image_checksum_fail-png')
     100    tests.add('failures/expected/keyboard.html',
     101                keyboard=True)
     102    tests.add('failures/expected/missing_check.html',
     103                expected_checksum=None)
     104    tests.add('failures/expected/missing_image.html',
     105                expected_image=None)
     106    tests.add('failures/expected/missing_text.html',
     107                expected_text=None)
     108    tests.add('failures/expected/newlines_leading.html',
     109                expected_text="\nfoo\n",
     110                actual_text="foo\n")
     111    tests.add('failures/expected/newlines_trailing.html',
     112                expected_text="foo\n\n",
     113                actual_text="foo\n")
     114    tests.add('failures/expected/newlines_with_excess_CR.html',
     115                expected_text="foo\r\r\r\n",
     116                actual_text="foo\n")
     117    tests.add('failures/expected/text.html',
     118                actual_text='text_fail-png')
     119    tests.add('failures/unexpected/crash.html', crash=True)
     120    tests.add('failures/unexpected/text-image-checksum.html',
     121                actual_text='text-image-checksum_fail-txt',
     122                actual_checksum='text-image-checksum_fail-checksum')
     123    tests.add('failures/unexpected/timeout.html', timeout=True)
     124    tests.add('http/tests/passes/text.html')
     125    tests.add('http/tests/ssl/text.html')
     126    tests.add('passes/error.html', error='stuff going to stderr')
     127    tests.add('passes/image.html')
     128    tests.add('passes/platform_image.html')
     129    # Text output files contain "\r\n" on Windows.  This may be
     130    # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling.
     131    tests.add('passes/text.html',
     132                expected_text='\nfoo\n\n',
     133                actual_text='\nfoo\r\n\r\r\n')
     134    tests.add('websocket/tests/passes/text.html')
     135    return tests
     136
     137
     138# Here we use a non-standard location for the layout tests, to ensure that
     139# this works. The path contains a '.' in the name because we've seen bugs
     140# related to this before.
     141
     142LAYOUT_TEST_DIR = '/test.checkout/LayoutTests'
     143
     144
     145# Here we synthesize an in-memory filesystem from the test list
     146# in order to fully control the test output and to demonstrate that
     147# we don't need a real filesystem to run the tests.
     148
     149def unit_test_filesystem(test_list=None):
     150    """Return the FileSystem object used by the unit tests."""
     151    test_list = test_list or unit_test_list()
     152    files = {}
     153
     154    def add_file(files, test, suffix, contents):
     155        dirname = test.name[0:test.name.rfind('/')]
     156        base = test.base
     157        path = LAYOUT_TEST_DIR + '/' + dirname + '/' + base + suffix
     158        files[path] = contents
     159
     160    # Add each test and the expected output, if any.
     161    for test in test_list.tests.values():
     162        add_file(files, test, '.html', '')
     163        add_file(files, test, '-expected.txt', test.expected_text)
     164        add_file(files, test, '-expected.checksum', test.expected_checksum)
     165        add_file(files, test, '-expected.png', test.expected_image)
     166
     167    # Add the test_expectations file.
     168    files[LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt'] = """
    278169WONTFIX : failures/expected/checksum.html = IMAGE
    279170WONTFIX : failures/expected/crash.html = CRASH
    … …  
    294185"""
    295186
     187    return filesystem_mock.MockFileSystem(files)
     188
     189
     190class TestPort(base.Port):
     191    """Test implementation of the Port interface."""
     192
     193    def __init__(self, **kwargs):
     194        self._tests = unit_test_list()
     195        if 'filesystem' not in kwargs:
     196            kwargs['filesystem'] = unit_test_filesystem(self._tests)
     197        kwargs.setdefault('port_name', 'test')
     198        base.Port.__init__(self, **kwargs)
     199
     200    def baseline_path(self):
     201        return self._filesystem.join(self.layout_tests_dir(), 'platform',
     202                                     self.name() + self.version())
     203
     204    def baseline_search_path(self):
     205        return [self.baseline_path()]
     206
     207    def check_build(self, needs_http):
     208        return True
     209
     210    def diff_image(self, expected_contents, actual_contents,
     211                   diff_filename=None):
     212        diffed = actual_contents != expected_contents
     213        if diffed and diff_filename:
     214            self._filesystem.write_text_file(diff_filename,
     215                "< %s\n---\n> %s\n" % (expected_contents, actual_contents))
     216        return diffed
     217
     218    def layout_tests_dir(self):
     219        return LAYOUT_TEST_DIR
     220
     221    def name(self):
     222        return self._name
     223
     224    def _path_to_wdiff(self):
     225        return None
     226
     227    def results_directory(self):
     228        return '/tmp/' + self.get_option('results_directory')
     229
     230    def setup_test_run(self):
     231        pass
     232
     233    def create_driver(self, worker_number):
     234        return TestDriver(self, worker_number)
     235
     236    def start_http_server(self):
     237        pass
     238
     239    def start_websocket_server(self):
     240        pass
     241
     242    def stop_http_server(self):
     243        pass
     244
     245    def stop_websocket_server(self):
     246        pass
     247
    296248    def test_base_platform_names(self):
    297249        return ('mac', 'win')
     250
     251    def test_expectations(self):
     252        return self._filesystem.read_text_file(LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt')
    298253
    299254    def test_platform_name(self):
Note: See TracChangeset for help on using the changeset viewer.