Changeset 73204 in webkit


Ignore:
Timestamp:
Dec 2, 2010 2:53:47 PM (13 years ago)
Author:
mihaip@chromium.org
Message:

2010-11-30 Mihai Parparita <​mihaip@chromium.org>

Reviewed by Tony Chang.

Rebaseline server: add updating of baselines
​https://bugs.webkit.org/show_bug.cgi?id=50305

Implements updating of baselines, where we copy -actual.* files over
the current -expected.* files. To do this, we need a
_get_actual_result_files method to get test results files and a
_rebaseline_test method to actually do the file copy and SCM operation.
_rebaseline_test logs output into a buffer, this is useful for both
showing result in the UI and for unit tests.

To make passing around of the various test environment properties
(results directory, filesystem, SCM, etc) easier, add a TestConfig
class for them.

Moving of existing baselines is not implemented yet, this patch is big
enough as it is.

  • Scripts/webkitpy/common/system/filesystem.py:
  • Scripts/webkitpy/common/system/filesystem_mock.py:
  • Scripts/webkitpy/tool/commands/data/rebaselineserver/queue.js:
  • Scripts/webkitpy/tool/commands/rebaselineserver.py:
  • Scripts/webkitpy/tool/commands/rebaselineserver_unittest.py:
  • Scripts/webkitpy/tool/mocktool.py:
Location:
trunk/WebKitTools
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/WebKitTools/ChangeLog

    r73194 r73204  
     12010-11-30  Mihai Parparita  <mihaip@chromium.org>
     2
     3        Reviewed by Tony Chang.
     4
     5        Rebaseline server: add updating of baselines
     6        https://bugs.webkit.org/show_bug.cgi?id=50305
     7       
     8        Implements updating of baselines, where we copy -actual.* files over
     9        the current -expected.* files. To do this, we need a
     10        _get_actual_result_files method to get test results files and a
     11        _rebaseline_test method to actually do the file copy and SCM operation.
     12        _rebaseline_test logs output into a buffer, this is useful for both
     13        showing result in the UI and for unit tests.
     14       
     15        To make passing around of the various test environment properties
     16        (results directory, filesystem, SCM, etc) easier, add a TestConfig
     17        class for them.
     18       
     19        Moving of existing baselines is not implemented yet, this patch is big
     20        enough as it is.
     21
     22        * Scripts/webkitpy/common/system/filesystem.py:
     23        * Scripts/webkitpy/common/system/filesystem_mock.py:
     24        * Scripts/webkitpy/tool/commands/data/rebaselineserver/queue.js:
     25        * Scripts/webkitpy/tool/commands/rebaselineserver.py:
     26        * Scripts/webkitpy/tool/commands/rebaselineserver_unittest.py:
     27        * Scripts/webkitpy/tool/mocktool.py:
     28
    1292010-12-02  Brent Fulgham  <bfulgham@webkit.org>
    230

  • trunk/WebKitTools/Scripts/webkitpy/common/system/filesystem.py

    r71383 r73204  
    3434import errno
    3535import os
     36import shutil
    3637import tempfile
    3738
    … …  
    4647        """Return whether the path exists in the filesystem."""
    4748        return os.path.exists(path)
     49
     50    def isfile(self, path):
     51        """Return whether the path refers to a file."""
     52        return os.path.isfile(path)
    4853
    4954    def isdir(self, path):
    … …  
    116121        with codecs.open(path, 'w', 'utf8') as f:
    117122            f.write(contents)
     123
     124    def copyfile(self, source, destination):
     125        """Copies the contents of the file at the given path to the destination
     126        path."""
     127        shutil.copyfile(source, destination)

  • trunk/WebKitTools/Scripts/webkitpy/common/system/filesystem_mock.py

    r72940 r73204  
    3333
    3434class MockFileSystem(object):
    35     def __init__(self, files={}):
     35    def __init__(self, files=None):
    3636        """Initializes a "mock" filesystem that can be used to completely
    3737        stub out a filesystem.
    … …  
    4242                not exist.
    4343        """
    44         self.files = files
     44        self.files = files or {}
    4545
    4646    def exists(self, path):
     47        return self.isfile(path) or self.isdir(path)
     48
     49    def isfile(self, path):
     50        return path in self.files and self.files[path] is not None
     51
     52    def isdir(self, path):
    4753        if path in self.files:
    48             return self.files[path] is not None
    49         return False
     54            return False
     55        if not path.endswith('/'):
     56            path += '/'
     57        return any(f.startswith(path) for f in self.files)
    5058
    5159    def join(self, *comps):
    5260        return '/'.join(comps)
     61
     62    def listdir(self, path):
     63        if not self.isdir(path):
     64            raise OSError("%s is not a directory" % path)
     65
     66        if not path.endswith('/'):
     67            path += '/'
     68
     69        dirs = []
     70        files = []
     71        for f in self.files:
     72            if self.exists(f) and f.startswith(path):
     73                remaining = f[len(path):]
     74                if '/' in remaining:
     75                    dir = remaining[:remaining.index('/')]
     76                    if not dir in dirs:
     77                        dirs.append(dir)
     78                else:
     79                    files.append(remaining)
     80        return dirs + files
    5381
    5482    def maybe_make_directory(self, *path):
    … …  
    7098    def write_binary_file(self, path, contents):
    7199        self.files[path] = contents
     100
     101    def copyfile(self, source, destination):
     102        if not self.exists(source):
     103            raise IOError(errno.ENOENT, source, os.strerror(errno.ENOENT))
     104        if self.isdir(source):
     105            raise IOError(errno.EISDIR, source, os.strerror(errno.ISDIR))
     106        if self.isdir(destination):
     107            raise IOError(errno.EISDIR, destination, os.strerror(errno.ISDIR))
     108
     109        self.files[destination] = self.files[source]

  • trunk/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/queue.js

    r72640 r73204  
    147147    var baselineMoveTo = getSelectValue('baseline-move-to');
    148148
    149     // FIXME: actually rebaseline
    150     log('Rebaselining ' + testName + ' for platform ' + baselineTarget + '...');
    151     var test = results.tests[testName];
    152     this._removeTest(testName);
    153     this._inProgressRebaselineCount--;
    154     test.state = STATE_REBASELINE_SUCCEEDED;
    155     updateState();
    156     log('Rebaselined add test with state ' + test.state + ' to queue',
    157         log.SUCCESS);
     149    var xhr = new XMLHttpRequest();
     150    xhr.open('POST',
     151        '/rebaseline?test=' + encodeURIComponent(testName) +
     152        '&baseline-target=' + encodeURIComponent(baselineTarget) +
     153        '&baseline-move-to=' + encodeURIComponent(baselineMoveTo));
     154
     155    var self = this;
     156    function handleResponse(logType, newState) {
     157        log(xhr.responseText, logType);
     158        self._removeTest(testName);
     159        self._inProgressRebaselineCount--;
     160        results.tests[testName].state = newState;
     161        updateState();
     162    }
     163
     164    function handleSuccess() {
     165        handleResponse(log.SUCCESS, STATE_REBASELINE_SUCCEEDED);
     166    }
     167    function handleFailure() {
     168        handleResponse(log.ERROR, STATE_REBASELINE_FAILED);
     169    }
     170
     171    xhr.addEventListener('load', function() {
     172      if (xhr.status < 400) {
     173          handleSuccess();
     174      } else {
     175          handleFailure();
     176      }
     177    });
     178    xhr.addEventListener('error', handleFailure);
     179
     180    xhr.send();
    158181};

  • trunk/WebKitTools/Scripts/webkitpy/tool/commands/rebaselineserver.py

    r72939 r73204  
    3434import codecs
    3535import datetime
     36import fnmatch
    3637import mimetypes
    3738import os
    … …  
    5758
    5859class RebaselineHTTPServer(BaseHTTPServer.HTTPServer):
    59     def __init__(self, httpd_port, results_directory, results_json, platforms_json):
     60    def __init__(self, httpd_port, test_config, results_json, platforms_json):
    6061        BaseHTTPServer.HTTPServer.__init__(self, ("", httpd_port), RebaselineHTTPRequestHandler)
    61         self.results_directory = results_directory
     62        self.test_config = test_config
    6263        self.results_json = results_json
    6364        self.platforms_json = platforms_json
    … …  
    113114        self._serve_file(os.path.join(
    114115            RebaselineHTTPRequestHandler.STATIC_FILE_DIRECTORY, static_path))
     116
     117    def rebaseline(self):
     118        test = self.query['test'][0]
     119        baseline_target = self.query['baseline-target'][0]
     120        baseline_move_to = self.query['baseline-move-to'][0]
     121        test_json = self.server.results_json['tests'][test]
     122
     123        if test_json['state'] != STATE_NEEDS_REBASELINE:
     124            self.send_error(400, "Test %s is in unexpected state: %s" %
     125                (test, test_json["state"]))
     126            return
     127
     128        log = []
     129        success = _rebaseline_test(
     130            test,
     131            baseline_target,
     132            baseline_move_to,
     133            self.server.test_config,
     134            log=lambda l: log.append(l))
     135
     136        if success:
     137            test_json['state'] = STATE_REBASELINE_SUCCEEDED
     138            self.send_response(200)
     139        else:
     140            test_json['state'] = STATE_REBASELINE_FAILED
     141            self.send_response(500)
     142
     143        self.send_header('Content-type', 'text/plain')
     144        self.end_headers()
     145        self.wfile.write('\n'.join(log))
    115146
    116147    def quitquitquit(self):
    … …  
    144175            file_name = test_name + '-diff.txt'
    145176
    146         file_path = os.path.join(self.server.results_directory, file_name)
     177        file_path = os.path.join(self.server.test_config.results_directory, file_name)
    147178
    148179        # Let results be cached for 60 seconds, so that they can be pre-fetched
    … …  
    184215
    185216
    186 def _get_test_baselines(test_file, test_port, layout_tests_directory, platforms, filesystem):
     217class TestConfig(object):
     218    def __init__(self, test_port, layout_tests_directory, results_directory, platforms, filesystem, scm):
     219        self.test_port = test_port
     220        self.layout_tests_directory = layout_tests_directory
     221        self.results_directory = results_directory
     222        self.platforms = platforms
     223        self.filesystem = filesystem
     224        self.scm = scm
     225
     226
     227def _get_actual_result_files(test_file, test_config):
     228    test_name, _ = os.path.splitext(test_file)
     229    test_directory = os.path.dirname(test_file)
     230
     231    test_results_directory = test_config.filesystem.join(
     232        test_config.results_directory, test_directory)
     233    actual_pattern = os.path.basename(test_name) + '-actual.*'
     234    actual_files = []
     235    for filename in test_config.filesystem.listdir(test_results_directory):
     236        if fnmatch.fnmatch(filename, actual_pattern):
     237            actual_files.append(filename)
     238    actual_files.sort()
     239    return tuple(actual_files)
     240
     241
     242def _rebaseline_test(test_file, baseline_target, baseline_move_to, test_config, log):
     243    test_name, _ = os.path.splitext(test_file)
     244    test_directory = os.path.dirname(test_name)
     245
     246    log('Rebaselining %s...' % test_name)
     247
     248    actual_result_files = _get_actual_result_files(test_file, test_config)
     249    filesystem = test_config.filesystem
     250    scm = test_config.scm
     251    layout_tests_directory = test_config.layout_tests_directory
     252    results_directory = test_config.results_directory
     253    target_expectations_directory = filesystem.join(
     254        layout_tests_directory, 'platform', baseline_target, test_directory)
     255    test_results_directory = test_config.filesystem.join(
     256        test_config.results_directory, test_directory)
     257
     258    # If requested, move current baselines out
     259    current_baselines = _get_test_baselines(test_file, test_config)
     260    if baseline_target in current_baselines and baseline_move_to != 'none':
     261        log('  Moving current %s baselines to %s' %
     262            (baseline_target, baseline_move_to))
     263        log('    FIXME: Add support for moving existing baselines')
     264        return False
     265
     266    log('  Updating baselines for %s' % baseline_target)
     267    for source_file in actual_result_files:
     268        source_path = filesystem.join(test_results_directory, source_file)
     269        destination_file = source_file.replace('-actual', '-expected')
     270        destination_path = filesystem.join(
     271            target_expectations_directory, destination_file)
     272        filesystem.copyfile(source_path, destination_path)
     273        exit_code = scm.add(destination_path, return_exit_code=True)
     274        if exit_code:
     275            log('    Could not update %s in SCM, exit code %d' %
     276                (destination_file, exit_code))
     277            return False
     278        else:
     279            log('    Updated %s' % destination_file)
     280
     281    return True
     282
     283
     284def _get_test_baselines(test_file, test_config):
    187285    class AllPlatformsPort(WebKitPort):
    188286        def __init__(self):
    189             WebKitPort.__init__(self, filesystem=filesystem)
     287            WebKitPort.__init__(self, filesystem=test_config.filesystem)
    190288            self._platforms_by_directory = dict(
    191                 [(self._webkit_baseline_path(p), p) for p in platforms])
     289                [(self._webkit_baseline_path(p), p) for p in test_config.platforms])
    192290
    193291        def baseline_search_path(self):
    … …  
    197295            return self._platforms_by_directory[directory]
    198296
    199     test_path = filesystem.join(layout_tests_directory, test_file)
     297    test_path = test_config.filesystem.join(
     298        test_config.layout_tests_directory, test_file)
    200299
    201300    all_platforms_port = AllPlatformsPort()
    … …  
    203302    all_test_baselines = {}
    204303    for baseline_extension in ('.txt', '.checksum', '.png'):
    205         test_baselines = test_port.expected_baselines(
     304        test_baselines = test_config.test_port.expected_baselines(
    206305            test_path, baseline_extension)
    207306        baselines = all_platforms_port.expected_baselines(
    … …  
    210309            if not platform_directory:
    211310                continue
    212             if platform_directory == layout_tests_directory:
     311            if platform_directory == test_config.layout_tests_directory:
    213312                platform = 'base'
    214313            else:
    … …  
    219318                platform_directory, expected_filename) in test_baselines
    220319            platform_baselines[baseline_extension] = was_used_for_test
    221        
     320
    222321    return all_test_baselines
    223322
    … …  
    248347        platforms = filesystem.listdir(
    249348            filesystem.join(layout_tests_directory, 'platform'))
     349        test_config = TestConfig(
     350            port,
     351            layout_tests_directory,
     352            results_directory,
     353            platforms,
     354            filesystem,
     355            self._tool.scm())
    250356
    251357        print 'Gathering current baselines...'
    … …  
    253359            test_json['state'] = STATE_NEEDS_REBASELINE
    254360            test_path = filesystem.join(layout_tests_directory, test_file)
    255             test_json['baselines'] = _get_test_baselines(
    256                 test_file, port, layout_tests_directory, platforms, filesystem)
     361            test_json['baselines'] = _get_test_baselines(test_file, test_config)
    257362
    258363        server_url = "http://localhost:%d/" % options.httpd_port
    … …  
    266371        httpd = RebaselineHTTPServer(
    267372            httpd_port=options.httpd_port,
    268             results_directory=results_directory,
     373            test_config=test_config,
    269374            results_json=results_json,
    270375            platforms_json={

  • trunk/WebKitTools/Scripts/webkitpy/tool/commands/rebaselineserver_unittest.py

    r72939 r73204  
    3333from webkitpy.layout_tests.port.webkit import WebKitPort
    3434from webkitpy.tool.commands import rebaselineserver
     35from webkitpy.tool.mocktool import MockSCM
     36
     37
     38class RebaselineTestTest(unittest.TestCase):
     39    def test_text_rebaseline(self):
     40        self._assertRebaseline(
     41            test_files=(
     42                'fast/text-expected.txt',
     43                'platform/mac/fast/text-expected.txt',
     44            ),
     45            results_files=(
     46                'fast/text-actual.txt',
     47            ),
     48            test_name='fast/text.html',
     49            baseline_target='mac',
     50            baseline_move_to='none',
     51            expected_success=True,
     52            expected_log=[
     53                'Rebaselining fast/text...',
     54                '  Updating baselines for mac',
     55                '    Updated text-expected.txt',
     56            ])
     57
     58    def test_text_rebaseline_move_no_op(self):
     59        self._assertRebaseline(
     60            test_files=(
     61                'fast/text-expected.txt',
     62                'platform/win/fast/text-expected.txt',
     63            ),
     64            results_files=(
     65                'fast/text-actual.txt',
     66            ),
     67            test_name='fast/text.html',
     68            baseline_target='mac',
     69            baseline_move_to='mac-leopard',
     70            expected_success=True,
     71            expected_log=[
     72                'Rebaselining fast/text...',
     73                '  Updating baselines for mac',
     74                '    Updated text-expected.txt',
     75            ])
     76
     77    def test_text_rebaseline_move(self):
     78        self._assertRebaseline(
     79            test_files=(
     80                'fast/text-expected.txt',
     81                'platform/mac/fast/text-expected.txt',
     82            ),
     83            results_files=(
     84                'fast/text-actual.txt',
     85            ),
     86            test_name='fast/text.html',
     87            baseline_target='mac',
     88            baseline_move_to='mac-leopard',
     89            expected_success=False,
     90            expected_log=[
     91                'Rebaselining fast/text...',
     92                '  Moving current mac baselines to mac-leopard',
     93                '    FIXME: Add support for moving existing baselines',
     94            ])
     95
     96    def test_image_rebaseline(self):
     97        self._assertRebaseline(
     98            test_files=(
     99                'fast/image-expected.txt',
     100                'platform/mac/fast/image-expected.png',
     101                'platform/mac/fast/image-expected.checksum',
     102            ),
     103            results_files=(
     104                'fast/image-actual.png',
     105                'fast/image-actual.checksum',
     106            ),
     107            test_name='fast/image.html',
     108            baseline_target='mac',
     109            baseline_move_to='none',
     110            expected_success=True,
     111            expected_log=[
     112                'Rebaselining fast/image...',
     113                '  Updating baselines for mac',
     114                '    Updated image-expected.checksum',
     115                '    Updated image-expected.png',
     116            ])
     117
     118    def _assertRebaseline(self, test_files, results_files, test_name, baseline_target, baseline_move_to, expected_success, expected_log):
     119        log = []
     120        test_config = get_test_config(test_files, results_files)
     121        success = rebaselineserver._rebaseline_test(
     122            test_name,
     123            baseline_target,
     124            baseline_move_to,
     125            test_config,
     126            log=lambda l: log.append(l))
     127        self.assertEqual(expected_log, log)
     128        self.assertEqual(expected_success, success)
     129
     130
     131class GetActualResultFilesTest(unittest.TestCase):
     132    def test(self):
     133        test_config = get_test_config(result_files=(
     134            'fast/text-actual.txt',
     135            'fast2/text-actual.txt',
     136            'fast/text2-actual.txt',
     137            'fast/text-notactual.txt',
     138        ))
     139        self.assertEqual(
     140            ('text-actual.txt',),
     141            rebaselineserver._get_actual_result_files(
     142                'fast/text.html', test_config))
    35143
    36144
    … …  
    80188
    81189    def _assertBaselines(self, test_files, test_name, expected_baselines):
    82         layout_tests_directory = base.Port().layout_tests_dir()
    83         mock_filesystem = filesystem_mock.MockFileSystem()
    84         for file in test_files + (test_name,):
    85             file_path = mock_filesystem.join(layout_tests_directory, file)
    86             mock_filesystem.files[file_path] = ''
    87 
    88         class TestMacPort(WebKitPort):
    89             def __init__(self):
    90                 WebKitPort.__init__(self, filesystem=mock_filesystem)
    91                 self._name = 'mac'
    92 
    93190        actual_baselines = rebaselineserver._get_test_baselines(
    94             test_name,
    95             TestMacPort(),
    96             layout_tests_directory,
    97             ('mac', 'win', 'linux'),
    98             mock_filesystem)
     191            test_name, get_test_config(test_files))
    99192        self.assertEqual(expected_baselines, actual_baselines)
     193
     194
     195def get_test_config(test_files=[], result_files=[]):
     196    layout_tests_directory = base.Port().layout_tests_dir()
     197    results_directory = '/WebKitBuild/Debug/layout-test-results'
     198    mock_filesystem = filesystem_mock.MockFileSystem()
     199    for file in test_files:
     200        file_path = mock_filesystem.join(layout_tests_directory, file)
     201        mock_filesystem.files[file_path] = ''
     202    for file in result_files:
     203        file_path = mock_filesystem.join(results_directory, file)
     204        mock_filesystem.files[file_path] = ''
     205
     206    class TestMacPort(WebKitPort):
     207        def __init__(self):
     208            WebKitPort.__init__(self, filesystem=mock_filesystem)
     209            self._name = 'mac'
     210
     211    return rebaselineserver.TestConfig(
     212        TestMacPort(),
     213        layout_tests_directory,
     214        results_directory,
     215        ('mac', 'win', 'linux'),
     216        mock_filesystem,
     217        MockSCM())

  • trunk/WebKitTools/Scripts/webkitpy/tool/mocktool.py

    r71580 r73204  
    450450        return "49824"
    451451
     452    def add(self, destination_path, return_exit_code=False):
     453        if return_exit_code:
     454            return 0
     455
    452456
    453457class MockCheckout(object):
Note: See TracChangeset for help on using the changeset viewer.