Changeset 261607 in webkit

Timestamp:

May 13, 2020 1:57:55 AM (4 years ago)

Author:

Philippe Normand

Message:

[Flatpak SDK] Craft a custom sccache config file from SDK toolchains
​https://bugs.webkit.org/show_bug.cgi?id=211781

Reviewed by Carlos Alberto Lopez Perez.

Optionally generate a sccache config file and use it when building WebKit.

• Scripts/webkitpy/thirdparty/init.py:

(AutoinstallImportHook.find_module):
(AutoinstallImportHook._install_toml):

• flatpak/flatpakutils.py:

(WebkitFlatpak.load_from_args):
(WebkitFlatpak.init):
(WebkitFlatpak.clean_args):
(WebkitFlatpak.host_path_to_sandbox_path):
(WebkitFlatpak.run_in_sandbox):
(WebkitFlatpak.main):
(WebkitFlatpak.save_config):
(WebkitFlatpak):
(WebkitFlatpak.pack_toolchain):
(WebkitFlatpak.setup_icecc): Deleted.

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/thirdparty/__init__.py (modified) (2 diffs)
• flatpak/flatpakutils.py (modified) (9 diffs)

trunk/Tools/ChangeLog

@@ +1 @@
+2020-05-13  Philippe Normand  <pnormand@igalia.com>
+
+        [Flatpak SDK] Craft a custom sccache config file from SDK toolchains
+        https://bugs.webkit.org/show_bug.cgi?id=211781
+
+        Reviewed by Carlos Alberto Lopez Perez.
+
+        Optionally generate a sccache config file and use it when building WebKit.
+
+        * Scripts/webkitpy/thirdparty/__init__.py:
+        (AutoinstallImportHook.find_module):
+        (AutoinstallImportHook._install_toml):
+        * flatpak/flatpakutils.py:
+        (WebkitFlatpak.load_from_args):
+        (WebkitFlatpak.__init__):
+        (WebkitFlatpak.clean_args):
+        (WebkitFlatpak.host_path_to_sandbox_path):
+        (WebkitFlatpak.run_in_sandbox):
+        (WebkitFlatpak.main):
+        (WebkitFlatpak.save_config):
+        (WebkitFlatpak):
+        (WebkitFlatpak.pack_toolchain):
+        (WebkitFlatpak.setup_icecc): Deleted.
+
 2020-05-13  Víctor Manuel Jáquez Leal  <vjaquez@igalia.com>
 

trunk/Tools/Scripts/webkitpy/thirdparty/__init__.py

@@ -129 +129 @@
         elif '.html5lib' in fullname:
             self._install_html5lib()
+        elif '.toml' in fullname:
+            self._install_toml()
 
     def _install_six(self):
@@ -324 +326 @@
             self.install_binary(full_firefox_url, 'geckodriver')
 
+    def _install_toml(self):
+        toml_dir = self._fs.join(_AUTOINSTALLED_DIR, "toml")
+        installer = AutoInstaller(prepend_to_search_path=True, target_dir=toml_dir)
+        installer.install(url="https://files.pythonhosted.org/packages/b9/19/5cbd78eac8b1783671c40e34bb0fa83133a06d340a38b55c645076d40094/toml-0.10.0.tar.gz", url_subpath="toml-0.10.0/toml")
+
     def _install(self, url, url_subpath=None, target_name=None):
         installer = AutoInstaller(target_dir=_AUTOINSTALLED_DIR)

trunk/Tools/flatpak/flatpakutils.py

@@ -38 +38 @@
 from webkitpy.port.factory import PortFactory
 from webkitpy.common.system.logutils import configure_logging
+import webkitpy.thirdparty.autoinstalled.toml
+import toml
 
 try:
@@ -418 +420 @@
                              dest="user_command")
         general.add_argument('--available', action='store_true', dest="check_available", help='Check if required dependencies are available.'),
-        general.add_argument("--use-icecream", dest="use_icecream", help="Use the distributed icecream (icecc) compiler.", action="store_true")
-        general.add_argument("-r", "--regenerate-toolchains", dest="regenerate_toolchains", action="store_true",
+        general.add_argument("--repo", help="Filesystem absolute path to the Flatpak repository to use", dest="user_repo")
+
+        distributed_build_options = parser.add_argument_group("Distributed building")
+        distributed_build_options.add_argument("--use-icecream", dest="use_icecream", help="Use the distributed icecream (icecc) compiler.", action="store_true")
+        distributed_build_options.add_argument("-r", "--regenerate-toolchains", dest="regenerate_toolchains", action="store_true",
                              help="Regenerate IceCC distribuable toolchain archives")
-        general.add_argument("--repo", help="Filesystem absolute path to the Flatpak repository to use", dest="user_repo")
+        distributed_build_options.add_argument("-t", "--sccache-token", dest="sccache_token",
+                                               help="sccache authentication token")
+        distributed_build_options.add_argument("-s", "--sccache-scheduler", dest="sccache_scheduler", default='https://sccache.igalia.com',
+                                               help="sccache scheduler URL")
 
         debugoptions = parser.add_argument_group("Debugging")
@@ -476 +484 @@
         self.icc_version = {}
         self.regenerate_toolchains = False
+        self.sccache_token = ""
+        self.sccache_scheduler = ""
 
     def execute_command(self, args, stdout=None, stderr=None):
@@ -521 +531 @@
         self.build_path = os.path.join(self.build_root, self.platform, self.build_type)
         self.config_file = os.path.join(self.flatpak_build_path, 'webkit_flatpak_config.json')
+        self.sccache_config_file = os.path.join(self.flatpak_build_path, 'sccache.toml')
 
         Console.quiet = self.quiet
@@ -607 +618 @@
                 return True
         return False
+
+    def host_path_to_sandbox_path(self, host_path):
+        # For now this supports only files in the WebKit path
+        return host_path.replace(self.source_root, self.sandbox_source_root)
 
     def run_in_sandbox(self, *args, **kwargs):
@@ -769 +784 @@
             flatpak_command.append(share_network_option)
 
+            if os.path.isfile(self.sccache_config_file) and not self.regenerate_toolchains and "SCCACHE_CONF" not in os.environ.keys():
+                sandbox_environment["SCCACHE_CONF"] = self.host_path_to_sandbox_path(self.sccache_config_file)
+
         override_sccache_server_port = os.environ.get("WEBKIT_SCCACHE_SERVER_PORT")
         if override_sccache_server_port:
@@ -837 +855 @@
         if regenerate_toolchains:
             self.icc_version = {}
-            self.setup_icecc("gcc", "g++")
-            self.setup_icecc("clang", "clang++")
-            self.save_config()
+            toolchains = self.pack_toolchain(("gcc", "g++"), {"/usr/bin/c++": "/usr/bin/g++"})
+            toolchains.extend(self.pack_toolchain(("clang", "clang++"), {"/usr/bin/clang++": "/usr/bin/clang++"}))
+            self.save_config(toolchains)
 
         return self.setup_dev_env()
@@ -855 +873 @@
         return os.path.exists(self.flatpak_build_path)
 
-    def save_config(self):
+    def save_config(self, toolchains):
         with open(self.config_file, 'w') as config:
             json_config = {'icecc_version': self.icc_version}
             json.dump(json_config, config)
 
-    def setup_icecc(self, *compilers):
+        if not self.sccache_token:
+            Console.message("No authentication token provided. Re-run this with the -t option if an sccache token was provided to you. Skipping sccache configuration for now.")
+            return
+
+        with open(self.sccache_config_file, 'w') as config:
+            sccache_config = {'dist': {'scheduler_url': self.sccache_scheduler,
+                                       'auth': {'type': 'token',
+                                                'token': self.sccache_token},
+                                       'toolchains': toolchains}}
+            toml.dump(sccache_config, config)
+            Console.message("Created %s sccache config file. It will automatically be used when building WebKit", self.sccache_config_file)
+
+    def pack_toolchain(self, compilers, path_mapping):
         with tempfile.NamedTemporaryFile() as tmpfile:
             command = ['icecc', '--build-native']
@@ -876 +906 @@
             Console.message("Created %s self-contained toolchain archive", archive_filename)
 
+            sccache_toolchains = []
+            for (compiler_executable, archive_compiler_executable) in path_mapping.iteritems():
+                item = {'type': 'path_override',
+                        'compiler_executable': compiler_executable,
+                        'archive': archive_filename,
+                        'archive_compiler_executable': archive_compiler_executable}
+                sccache_toolchains.append(item)
+            return sccache_toolchains
+
     def setup_dev_env(self):
         if not os.path.exists(os.path.join(self.flatpak_build_path, "runtime", "org.webkit.Sdk")) or self.update: