Changeset 181014 in webkit

Timestamp:

Mar 4, 2015 1:02:33 PM (9 years ago)

Author:

youenn.fablet@crf.canon.fr

Message:

W3C test importer should use filesystem instead of os.walk
​https://bugs.webkit.org/show_bug.cgi?id=142085

Reviewed by Bem Jones-Bey.

Added FileSystem.dirs_under to remove the use of os.walk in test importer.
Added MockFileSystem.dirs_under and MockFileSystem.getsize to enable unit testing of test importer.
Added unit test for FileSystem.dirs_under and MockFileSystem.dirs_under.

Made use of FileSystem.dirs_under within TestImporter.find_importable_tests.
Added a unit test to check that test importer is now black-box testable using a Mock system.

• Scripts/webkitpy/common/system/filesystem.py:

(FileSystem.dirs_under): Returns a list of filtered sub-directories.
(FileSystem.dirs_under.filter_all):

• Scripts/webkitpy/common/system/filesystem_mock.py:

(MockFileSystem.dirs_under):
(MockFileSystem.dirs_under.filter_all):
(MockFileSystem.getsize):

• Scripts/webkitpy/common/system/filesystem_mock_unittest.py:

(MockFileSystemTest.test_dirs_under):
(MockFileSystemTest.test_dirs_under.filter_dir):

• Scripts/webkitpy/common/system/filesystem_unittest.py:

(RealFileSystemTest.test_sep):
(RealFileSystemTest):
(RealFileSystemTest.test_dirs_under):
(RealFileSystemTest.test_dirs_under.filter_this_dir):

• Scripts/webkitpy/w3c/test_importer.py:

(TestImporter.do_import):
(TestImporter.find_importable_tests.should_keep_subdir):
(TestImporter.find_importable_tests):
(TestImporter.should_keep_subdir): Deleted.
(TestImporter.should_skip_file): Deleted.

• Scripts/webkitpy/w3c/test_importer_unittest.py:

(TestImporterTest.test_import_dir_with_empty_init_py): Added test to ensure empty init.py are no longer empty once imported.

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/common/system/filesystem.py (modified) (1 diff)
• Scripts/webkitpy/common/system/filesystem_mock.py (modified) (2 diffs)
• Scripts/webkitpy/common/system/filesystem_mock_unittest.py (modified) (1 diff)
• Scripts/webkitpy/common/system/filesystem_unittest.py (modified) (1 diff)
• Scripts/webkitpy/w3c/test_importer.py (modified) (3 diffs)
• Scripts/webkitpy/w3c/test_importer_unittest.py (modified) (1 diff)

trunk/Tools/ChangeLog

@@ +1 @@
+2015-03-04  Youenn Fablet  <youenn.fablet@crf.canon.fr>
+
+        W3C test importer should use filesystem instead of os.walk
+        https://bugs.webkit.org/show_bug.cgi?id=142085
+
+        Reviewed by Bem Jones-Bey.
+
+        Added FileSystem.dirs_under to remove the use of os.walk in test importer.
+        Added MockFileSystem.dirs_under and MockFileSystem.getsize to enable unit testing of test importer.
+        Added unit test for FileSystem.dirs_under and MockFileSystem.dirs_under.
+
+        Made use of FileSystem.dirs_under within TestImporter.find_importable_tests.
+        Added a unit test to check that test importer is now black-box testable using a Mock system.
+
+        * Scripts/webkitpy/common/system/filesystem.py:
+        (FileSystem.dirs_under): Returns a list of filtered sub-directories.
+        (FileSystem.dirs_under.filter_all):
+        * Scripts/webkitpy/common/system/filesystem_mock.py:
+        (MockFileSystem.dirs_under):
+        (MockFileSystem.dirs_under.filter_all):
+        (MockFileSystem.getsize):
+        * Scripts/webkitpy/common/system/filesystem_mock_unittest.py:
+        (MockFileSystemTest.test_dirs_under):
+        (MockFileSystemTest.test_dirs_under.filter_dir):
+        * Scripts/webkitpy/common/system/filesystem_unittest.py:
+        (RealFileSystemTest.test_sep):
+        (RealFileSystemTest):
+        (RealFileSystemTest.test_dirs_under):
+        (RealFileSystemTest.test_dirs_under.filter_this_dir):
+        * Scripts/webkitpy/w3c/test_importer.py:
+        (TestImporter.do_import):
+        (TestImporter.find_importable_tests.should_keep_subdir):
+        (TestImporter.find_importable_tests):
+        (TestImporter.should_keep_subdir): Deleted.
+        (TestImporter.should_skip_file): Deleted.
+        * Scripts/webkitpy/w3c/test_importer_unittest.py:
+        (TestImporterTest.test_import_dir_with_empty_init_py): Added test to ensure empty __init__.py are no longer empty once imported.
+
 2015-03-04  Timothy Horton  <timothy_horton@apple.com>
 

trunk/Tools/Scripts/webkitpy/common/system/filesystem.py

@@ -78 +78 @@
         return os.path.exists(path)
 
+    def dirs_under(self, path, dir_filter=None):
+        """Return the list of all directories under the given path in topdown order.
+
+        Args:
+            dir_filter: if not None, the filter will be invoked
+                with the filesystem object and the path of each dirfound.
+                The dir is included in the result if the callback returns True.
+        """
+        def filter_all(fs, dirpath):
+            return True
+        dir_filter = dir_filter or filter_all
+
+        dirs = []
+        for (dirpath, dirnames, filenames) in os.walk(path):
+            if dir_filter(self, dirpath):
+                dirs.append(dirpath)
+        return dirs
+
     def files_under(self, path, dirs_to_skip=[], file_filter=None):
         """Return the list of all files under the given path in topdown order.

trunk/Tools/Scripts/webkitpy/common/system/filesystem_mock.py

@@ -124 +124 @@
         return self.isfile(path) or self.isdir(path)
 
+    def dirs_under(self, path, dirs_filter=None):
+        def filter_all(fs, dirpath):
+            return True
+
+        dirs_filter = dirs_filter or filter_all
+
+        dirs = []
+        for dirpath in self.dirs:
+            if not dirpath.startswith(path):
+                continue
+            if dirs_filter(self, dirpath):
+                dirs.append(dirpath)
+        return sorted(dirs)
+
+
     def files_under(self, path, dirs_to_skip=[], file_filter=None):
         def filter_all(fs, dirpath, basename):
@@ -158 +173 @@
     def getcwd(self):
         return self.cwd
+
+    def getsize(self, path):
+        if not self.isfile(path):
+            raise OSError("%s is not a file" % path)
+        return len(self.files[path])
 
     def glob(self, glob_string):

trunk/Tools/Scripts/webkitpy/common/system/filesystem_mock_unittest.py

@@ -83 +83 @@
                          'foo/../bar/baz',
                          '../foo')
+
+    def test_dirs_under(self):
+        FAKE_FILES = {
+            '/tests/test1.txt': '',
+            '/tests/test3/test2/test.txt': 'test',
+            '/tests/test2/test.txt': 'test'}
+        fs = filesystem_mock.MockFileSystem(files=FAKE_FILES)
+
+        self.assertEquals(fs.dirs_under('/tests'), ['/tests', '/tests/test2', '/tests/test3', '/tests/test3/test2'])
+
+        def filter_dir(fs, dirpath):
+            return fs.basename(dirpath) != 'test2'
+
+        self.assertEquals(fs.dirs_under('/tests', filter_dir), ['/tests', '/tests/test3'])

trunk/Tools/Scripts/webkitpy/common/system/filesystem_unittest.py

@@ -265 +265 @@
         self.assertEqual(fs.join("foo", "bar"),
                           os.path.join("foo", "bar"))
+
+    def test_dirs_under(self):
+        fs = FileSystem()
+        parentDir = fs.normpath(fs.join(self._this_dir, ".."))
+
+        self.assertTrue(self._this_dir in fs.dirs_under(parentDir))
+
+        def filter_no_dir(fs, dirpath):
+            return False
+        self.assertEqual(len(fs.dirs_under(parentDir, filter_no_dir)), 0)
+
+        def filter_this_dir(fs, dirpath):
+            return dirpath != self._this_dir
+        self.assertFalse(self._this_dir in fs.dirs_under(parentDir, filter_this_dir))

trunk/Tools/Scripts/webkitpy/w3c/test_importer.py

@@ -162 +162 @@
         self.import_tests()
 
-    def should_keep_subdir(self, root, subdir):
-        DIRS_TO_SKIP = ('work-in-progress', 'tools', 'support')
-        should_skip = subdir.startswith('.') or (root == self.source_directory and subdir in DIRS_TO_SKIP)
-        return not should_skip
-
     def should_skip_file(self, filename):
         # For some reason the w3c repo contains random perl scripts we don't care about.
@@ -176 +171 @@
 
     def find_importable_tests(self, directory):
-        # FIXME: use filesystem
-        for root, dirs, files in os.walk(directory):
+        def should_keep_subdir(filesystem, path):
+            subdir = path[len(directory):]
+            DIRS_TO_SKIP = ('work-in-progress', 'tools', 'support')
+            should_skip = filesystem.basename(subdir).startswith('.') or (subdir in DIRS_TO_SKIP)
+            return not should_skip
+
+        directories = self.filesystem.dirs_under(directory, should_keep_subdir)
+        for root in directories:
             _log.info('Scanning ' + root + '...')
             total_tests = 0
@@ -183 +184 @@
             jstests = 0
 
-            dirs[:] = [subdir for subdir in dirs if self.should_keep_subdir(root, subdir)]
-
             copy_list = []
 
-            for filename in files:
+            for filename in self.filesystem.listdir(root):
+                if self.filesystem.isdir(self.filesystem.join(root, filename)):
+                    continue
                 # FIXME: This block should really be a separate function, but the early-continues make that difficult.
 

trunk/Tools/Scripts/webkitpy/w3c/test_importer_unittest.py

@@ -75 +75 @@
             oc.restore_output()
 
+    def test_import_dir_with_empty_init_py(self):
+        FAKE_FILES = {
+            '/tests/csswg/test1/__init__.py': '',
+            '/tests/csswg/test2/__init__.py': 'NOTEMPTY',
+        }
+
+        host = MockHost()
+        host.filesystem = MockFileSystem(files=FAKE_FILES)
+
+        importer = TestImporter(host, FAKE_SOURCE_DIR, optparse.Values({"overwrite": False, 'destination': 'w3c', 'test_paths': ['/tests/csswg']}))
+        importer.do_import()
+
+        self.assertTrue(host.filesystem.exists("/mock-checkout/LayoutTests/w3c/test1/__init__.py"))
+        self.assertTrue(host.filesystem.exists("/mock-checkout/LayoutTests/w3c/test2/__init__.py"))
+        self.assertTrue(host.filesystem.getsize("/mock-checkout/LayoutTests/w3c/test1/__init__.py") > 0)
+
     # FIXME: Needs more tests.