Changeset 120740 in webkit


Ignore:
Timestamp:
Jun 19, 2012 11:27:29 AM (12 years ago)
Author:
dpranke@chromium.org
Message:

REGRESSION (NRWT): Results for new non-text-only tests are always put in the most-specific platform directory
https://bugs.webkit.org/show_bug.cgi?id=78127

Reviewed by Ryosuke Niwa.

Add the --add-platform-exceptions flag from ORWT, and ensure
that --new-baseline is equivalent to --reset-results
--add-platform-exceptions. Also fix the default logic for where to
put PNGs and render tree dumps if --new-test-results is true:
if --add-platform-exceptions is False, baselines should go in
the least-specific platform directory (e.g., platform/mac)
rather than the most-specific (platform/mac-snowleopard).

  • Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:

(SingleTestRunner):
(SingleTestRunner.run):
(SingleTestRunner._run_rebaseline):
(SingleTestRunner._add_missing_baselines):
(SingleTestRunner._location_for_new_baseline):
(SingleTestRunner._overwrite_baselines):
(SingleTestRunner._save_baseline_data):

  • Scripts/webkitpy/layout_tests/port/base.py:

(Port.baseline_path):
(Port):
(Port.baseline_platform_dir):
(Port.baseline_version_dir):

  • Scripts/webkitpy/layout_tests/run_webkit_tests.py:

(_set_up_derived_options):
(parse_args):

  • Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:

(RebaselineTest.assertBaselines):
(RebaselineTest.test_reset_results):
(RebaselineTest.test_missing_results):
(RebaselineTest.test_new_baseline):

Location:
trunk/Tools
Files:
5 edited

Legend:

Unmodified
Added
Removed
  • trunk/Tools/ChangeLog

    r120738 r120740  
     12012-06-19  Dirk Pranke  <dpranke@chromium.org>
     2
     3        REGRESSION (NRWT): Results for new non-text-only tests are always put in the most-specific platform directory
     4        https://bugs.webkit.org/show_bug.cgi?id=78127
     5
     6        Reviewed by Ryosuke Niwa.
     7
     8        Add the --add-platform-exceptions flag from ORWT, and ensure
     9        that --new-baseline is equivalent to --reset-results
     10        --add-platform-exceptions. Also fix the default logic for where to
     11        put PNGs and render tree dumps if --new-test-results is true:
     12        if --add-platform-exceptions is False, baselines should go in
     13        the least-specific platform directory (e.g., platform/mac)
     14        rather than the most-specific (platform/mac-snowleopard).
     15
     16        * Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
     17        (SingleTestRunner):
     18        (SingleTestRunner.run):
     19        (SingleTestRunner._run_rebaseline):
     20        (SingleTestRunner._add_missing_baselines):
     21        (SingleTestRunner._location_for_new_baseline):
     22        (SingleTestRunner._overwrite_baselines):
     23        (SingleTestRunner._save_baseline_data):
     24        * Scripts/webkitpy/layout_tests/port/base.py:
     25        (Port.baseline_path):
     26        (Port):
     27        (Port.baseline_platform_dir):
     28        (Port.baseline_version_dir):
     29        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
     30        (_set_up_derived_options):
     31        (parse_args):
     32        * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
     33        (RebaselineTest.assertBaselines):
     34        (RebaselineTest.test_reset_results):
     35        (RebaselineTest.test_missing_results):
     36        (RebaselineTest.test_new_baseline):
     37
    1382012-06-19  Dirk Pranke  <dpranke@chromium.org>
    239
  • trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py

    r116600 r120740  
    4747
    4848
    49 class SingleTestRunner:
     49class SingleTestRunner(object):
     50    (ALONGSIDE_TEST, PLATFORM_DIR, VERSION_DIR, UPDATE) = ('alongside', 'platform', 'version', 'update')
    5051
    5152    def __init__(self, options, port, driver, test_input, worker_name):
     
    9596    def run(self):
    9697        if self._reference_files:
    97             if self._port.get_option('no_ref_tests') or self._options.new_baseline or self._options.reset_results:
     98            if self._port.get_option('no_ref_tests') or self._options.reset_results:
    9899                result = TestResult(self._test_name)
    99100                result.type = test_expectations.SKIP
    100101                return result
    101102            return self._run_reftest()
    102         if self._options.new_baseline or self._options.reset_results:
     103        if self._options.reset_results:
    103104            return self._run_rebaseline()
    104105        return self._run_compare_test()
     
    122123        failures = self._handle_error(driver_output)
    123124        test_result_writer.write_test_result(self._filesystem, self._port, self._test_name, driver_output, None, failures)
    124         # FIXME: It the test crashed or timed out, it might be bettter to avoid
     125        # FIXME: It the test crashed or timed out, it might be better to avoid
    125126        # to write new baselines.
    126127        self._overwrite_baselines(driver_output)
     
    132133        missingImage = test_result.has_failure_matching_types(test_failures.FailureMissingImage, test_failures.FailureMissingImageHash)
    133134        if test_result.has_failure_matching_types(test_failures.FailureMissingResult):
    134             self._save_baseline_data(driver_output.text, ".txt", SingleTestRunner._render_tree_dump_pattern.match(driver_output.text))
     135            self._save_baseline_data(driver_output.text, '.txt', self._location_for_new_baseline(driver_output.text, '.txt'))
    135136        if test_result.has_failure_matching_types(test_failures.FailureMissingAudio):
    136             self._save_baseline_data(driver_output.audio, ".wav", generate_new_baseline=False)
     137            self._save_baseline_data(driver_output.audio, '.wav', self._location_for_new_baseline(driver_output.audio, '.wav'))
    137138        if missingImage:
    138             self._save_baseline_data(driver_output.image, ".png", generate_new_baseline=True)
     139            self._save_baseline_data(driver_output.image, '.png', self._location_for_new_baseline(driver_output.image, '.png'))
     140
     141    def _location_for_new_baseline(self, data, extension):
     142        if self._options.add_platform_exceptions:
     143            return self.VERSION_DIR
     144        if extension == '.png':
     145            return self.PLATFORM_DIR
     146        if extension == '.wav':
     147            return self.ALONGSIDE_TEST
     148        if extension == '.txt' and self._render_tree_dump_pattern.match(data):
     149            return self.PLATFORM_DIR
     150        return self.ALONGSIDE_TEST
    139151
    140152    def _overwrite_baselines(self, driver_output):
    141         # Although all DumpRenderTree output should be utf-8,
    142         # we do not ever decode it inside run-webkit-tests.  For some tests
    143         # DumpRenderTree may not output utf-8 text (e.g. webarchives).
    144         self._save_baseline_data(driver_output.text, ".txt", generate_new_baseline=self._options.new_baseline)
    145         self._save_baseline_data(driver_output.audio, ".wav", generate_new_baseline=self._options.new_baseline)
     153        location = self.VERSION_DIR if self._options.add_platform_exceptions else self.UPDATE
     154        self._save_baseline_data(driver_output.text, '.txt', location)
     155        self._save_baseline_data(driver_output.audio, '.wav', location)
    146156        if self._options.pixel_tests:
    147             self._save_baseline_data(driver_output.image, ".png", generate_new_baseline=self._options.new_baseline)
    148 
    149     def _save_baseline_data(self, data, modifier, generate_new_baseline=True):
    150         """Saves a new baseline file into the port's baseline directory.
    151 
    152         The file will be named simply "<test>-expected<modifier>", suitable for
    153         use as the expected results in a later run.
    154 
    155         Args:
    156           data: result to be saved as the new baseline
    157           modifier: type of the result file, e.g. ".txt" or ".png"
    158           generate_new_baseline: whether to enerate a new, platform-specific
    159             baseline, or update the existing one
    160         """
     157            self._save_baseline_data(driver_output.image, '.png', location)
     158
     159    def _save_baseline_data(self, data, extension, location):
    161160        if data is None:
    162161            return
    163162        port = self._port
    164163        fs = self._filesystem
    165         if generate_new_baseline:
    166             relative_dir = fs.dirname(self._test_name)
    167             baseline_path = port.baseline_path()
    168             output_dir = fs.join(baseline_path, relative_dir)
    169             output_file = fs.basename(fs.splitext(self._test_name)[0] + "-expected" + modifier)
    170             fs.maybe_make_directory(output_dir)
    171             output_path = fs.join(output_dir, output_file)
     164        if location == self.ALONGSIDE_TEST:
     165            output_dir = fs.dirname(port.abspath_for_test(self._test_name))
     166        elif location == self.VERSION_DIR:
     167            output_dir = fs.join(port.baseline_version_dir(), fs.dirname(self._test_name))
     168        elif location == self.PLATFORM_DIR:
     169            output_dir = fs.join(port.baseline_platform_dir(), fs.dirname(self._test_name))
     170        elif location == self.UPDATE:
     171            output_dir = fs.dirname(port.expected_filename(self._test_name, extension))
    172172        else:
    173             output_path = port.expected_filename(self._test_name, modifier)
    174 
    175         result_name = fs.relpath(output_path, port.layout_tests_dir())
    176         _log.info('Writing new expected result "%s"' % result_name)
     173            raise AssertionError('unrecognized baseline location: %s' % location)
     174
     175        fs.maybe_make_directory(output_dir)
     176        output_basename = fs.basename(fs.splitext(self._test_name)[0] + "-expected" + extension)
     177        output_path = fs.join(output_dir, output_basename)
     178        _log.info('Writing new expected result "%s"' % port.relative_test_filename(output_path))
    177179        port.update_baseline(output_path, data)
    178180
  • trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py

    r120738 r120740  
    166166    def baseline_path(self):
    167167        """Return the absolute path to the directory to store new baselines in for this port."""
     168        # FIXME: remove once all callers are calling either baseline_version_dir() or baseline_platform_dir()
     169        return self.baseline_version_dir()
     170
     171    def baseline_platform_dir(self):
     172        """Return the absolute path to the default (version-independent) platform-specific results."""
     173        return self._filesystem.join(self.layout_tests_dir(), 'platform', self.port_name)
     174
     175    def baseline_version_dir(self):
     176        """Return the absolute path to the platform-and-version-specific results."""
    168177        baseline_search_paths = self.get_option('additional_platform_directory', []) + self.baseline_search_path()
    169178        return baseline_search_paths[0]
  • trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

    r120646 r120740  
    168168        warnings.append("--ignore-metrics has no effect with --new-baselines or with --reset-results")
    169169
     170    if options.new_baseline:
     171        options.reset_results = True
     172        options.add_platform_exceptions = True
     173
    170174    return warnings
    171175
     
    278282        optparse.make_option("--build-directory",
    279283            help="Path to the directory under which build files are kept (should not include configuration)"),
     284        optparse.make_option("--add-platform-exceptions", action="store_true", default=False,
     285            help="Save generated results into the *most-specific-platform* directory rather than the *generic-platform* directory"),
    280286        optparse.make_option("--new-baseline", action="store_true",
    281287            default=False, help="Save generated results as new baselines "
    282                  "into the *platform* directory, overwriting whatever's "
    283                  "already there."),
     288                 "into the *most-specific-platform* directory, overwriting whatever's "
     289                 "already there. Equivalent to --reset-results --add-platform-exceptions"),
    284290        optparse.make_option("--reset-results", action="store_true",
    285291            default=False, help="Reset expectations to the "
  • trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py

    r120349 r120740  
    931931        for ext in extensions:
    932932            baseline = file + "-expected" + ext
    933             baseline_msg = 'Writing new expected result "%s"\n' % baseline[1:]
     933            baseline_msg = 'Writing new expected result "%s"\n' % baseline
    934934            self.assertTrue(any(f.find(baseline) != -1 for f in file_list))
    935935            self.assertContainsLine(err, baseline_msg)
     
    952952        self.assertEmpty(out)
    953953        self.assertEqual(len(file_list), 4)
    954         self.assertBaselines(file_list, "/passes/image", [".txt", ".png"], err)
    955         self.assertBaselines(file_list, "/failures/expected/missing_image", [".txt", ".png"], err)
     954        self.assertBaselines(file_list, "passes/image", [".txt", ".png"], err)
     955        self.assertBaselines(file_list, "failures/expected/missing_image", [".txt", ".png"], err)
    956956
    957957    def test_missing_results(self):
     
    970970        self.assertNotEmpty(out)
    971971        self.assertEqual(len(file_list), 6)
    972         self.assertBaselines(file_list, "/failures/unexpected/missing_text", [".txt"], err)
    973         self.assertBaselines(file_list, "/platform/test-mac-leopard/failures/unexpected/missing_image", [".png"], err)
    974         self.assertBaselines(file_list, "/platform/test-mac-leopard/failures/unexpected/missing_render_tree_dump", [".txt"], err)
     972        self.assertBaselines(file_list, "failures/unexpected/missing_text", [".txt"], err)
     973        self.assertBaselines(file_list, "platform/test/failures/unexpected/missing_image", [".png"], err)
     974        self.assertBaselines(file_list, "platform/test/failures/unexpected/missing_render_tree_dump", [".txt"], err)
    975975
    976976    def test_new_baseline(self):
    977         # Test that we update the platform expectations. If the expectation
    978         # is mssing, then create a new expectation in the platform dir.
     977        # Test that we update the platform expectations in the version-specific directories
     978        # for both existing and new baselines.
    979979        host = MockHost()
    980980        res, out, err, _ = logging_run(['--pixel-tests',
     
    989989        self.assertEqual(len(file_list), 4)
    990990        self.assertBaselines(file_list,
    991             "/platform/test-mac-leopard/passes/image", [".txt", ".png"], err)
     991            "platform/test-mac-leopard/passes/image", [".txt", ".png"], err)
    992992        self.assertBaselines(file_list,
    993             "/platform/test-mac-leopard/failures/expected/missing_image", [".txt", ".png"], err)
     993            "platform/test-mac-leopard/failures/expected/missing_image", [".txt", ".png"], err)
    994994
    995995
Note: See TracChangeset for help on using the changeset viewer.