Add support for REAPI execution from the sandbox

doc/source/format_declaring.rst

@@ -389,6 +389,24 @@ field in the ``Command`` uploaded. Whether this actually results in a building
 the element for the desired OS and architecture is dependent on the server
 having implemented these options the same as buildstream.
 
+.. code:: yaml
+
+   # Specify UNIX socket path for access to REAPI for (nested) remote execution
+   sandbox:
+     remote-apis-socket:
+       path: /run/reapi.sock
+
+Setting a path will add a UNIX socket to the sandbox that allows the use of
+`REAPI <https://github.com/bazelbuild/remote-apis>`_ clients such as
+`recc <https://buildgrid.gitlab.io/recc>`_.
+
+This enables more fine-grained caching of, e.g., individual compile commands
+to speed up rebuilds of elements with only small changes.
+
+This is supported with and without :ref:`remote execution <user_config_remote_execution>`.
+With remote execution configured, this additionally enables scaling out of,
+e.g., compile commands across a cluster of build machines.
+
 
 .. _format_dependencies:
 

src/buildstream/_cas/casdprocessmanager.py

@@ -45,9 +45,9 @@
 #
 # Minimum required version of buildbox-casd
 #
-_REQUIRED_CASD_MAJOR = 0
-_REQUIRED_CASD_MINOR = 0
-_REQUIRED_CASD_MICRO = 58
+_REQUIRED_CASD_MAJOR = 1
+_REQUIRED_CASD_MINOR = 2
+_REQUIRED_CASD_MICRO = 0
 
 
 # CASDProcessManager
@@ -76,7 +76,8 @@ def __init__(
         messenger,
         *,
         reserved=None,
-        low_watermark=None
+        low_watermark=None,
+        local_jobs=None
     ):
         os.makedirs(path, exist_ok=True)
 
@@ -104,6 +105,17 @@ def __init__(
         if protect_session_blobs:
             casd_args.append("--protect-session-blobs")
 
+        if local_jobs is not None:
+            try:
+                buildbox_run = utils._get_host_tool_internal("buildbox-run", search_subprojects_dir="buildbox")
+                casd_args.append("--buildbox-run={}".format(buildbox_run))
+                casd_args.append("--jobs={}".format(local_jobs))
+            except utils.ProgramNotFoundError:
+                # Not fatal as buildbox-run is not needed for remote execution
+                # and buildbox-casd local execution will never be used if
+                # buildbox-run is not available.
+                pass
+
         if remote_cache_spec:
             casd_args.append("--cas-remote={}".format(remote_cache_spec.url))
             if remote_cache_spec.instance_name:

src/buildstream/_context.py

@@ -531,6 +531,13 @@ def sourcecache(self) -> SourceCache:
 
         return self._sourcecache
 
+    @property
+    def effective_build_max_jobs(self) -> int:
+        # Based on some testing (mainly on AWS), maximum effective
+        # max-jobs value seems to be around 8-10 if we have enough cores
+        # users should set values based on workload and build infrastructure
+        return self.build_max_jobs or self.platform.get_cpu_count(8)
+
     # add_project():
     #
     # Add a project to the context.
@@ -727,6 +734,7 @@ def get_casd(self) -> CASDProcessManager:
 
             assert self.logdir is not None, "log_directory is required for casd"
             log_dir = os.path.join(self.logdir, "_casd")
+            assert self.sched_builders is not None, "builders configuration is required"
             self._casd = CASDProcessManager(
                 self.cachedir,
                 log_dir,
@@ -737,6 +745,7 @@ def get_casd(self) -> CASDProcessManager:
                 messenger=self.messenger,
                 reserved=self.config_cache_reserved,
                 low_watermark=self.config_cache_low_watermark,
+                local_jobs=self.sched_builders * self.effective_build_max_jobs,
             )
         return self._casd
 

src/buildstream/_project.py

@@ -1075,16 +1075,7 @@ def _load_pass(self, config, output, *, ignore_unknown=False):
 
         # Extend variables with automatic variables and option exports
         # Initialize it as a string as all variables are processed as strings.
-        # Based on some testing (mainly on AWS), maximum effective
-        # max-jobs value seems to be around 8-10 if we have enough cores
-        # users should set values based on workload and build infrastructure
-        if self._context.build_max_jobs == 0:
-            # User requested automatic max-jobs
-            platform = self._context.platform
-            output.base_variables["max-jobs"] = str(platform.get_cpu_count(8))
-        else:
-            # User requested explicit max-jobs setting
-            output.base_variables["max-jobs"] = str(self._context.build_max_jobs)
+        output.base_variables["max-jobs"] = str(self._context.effective_build_max_jobs)
 
         # Export options into variables, if that was requested
         output.options.export_variables(output.base_variables)

src/buildstream/sandbox/_config.py

@@ -34,6 +34,7 @@
 #    build_arch: A canonical machine architecture name, as defined by Platform.canonicalize_arch()
 #    build_uid: The UID for the sandbox process
 #    build_gid: The GID for the sandbox process
+#    remote_apis_socket_path: The path to a UNIX socket providing REAPI access for nested remote execution
 #
 # If the build_uid or build_gid is unspecified, then the underlying sandbox implementation
 # does not guarantee what UID/GID will be used, but generally UID/GID 0 will be used in a
@@ -45,12 +46,19 @@
 #
 class SandboxConfig:
     def __init__(
-        self, *, build_os: str, build_arch: str, build_uid: Optional[int] = None, build_gid: Optional[int] = None
+        self,
+        *,
+        build_os: str,
+        build_arch: str,
+        build_uid: Optional[int] = None,
+        build_gid: Optional[int] = None,
+        remote_apis_socket_path: Optional[str] = None
     ):
         self.build_os = build_os
         self.build_arch = build_arch
         self.build_uid = build_uid
         self.build_gid = build_gid
+        self.remote_apis_socket_path = remote_apis_socket_path
 
     # to_dict():
     #
@@ -87,6 +95,9 @@ def to_dict(self) -> Dict[str, Union[str, int]]:
         if self.build_gid is not None:
             sandbox_dict["build-gid"] = self.build_gid
 
+        if self.remote_apis_socket_path is not None:
+            sandbox_dict["remote-apis-socket-path"] = self.remote_apis_socket_path
+
         return sandbox_dict
 
     # new_from_node():
@@ -108,7 +119,7 @@ def to_dict(self) -> Dict[str, Union[str, int]]:
     #
     @classmethod
     def new_from_node(cls, config: "MappingNode[Node]", *, platform: Optional[Platform] = None) -> "SandboxConfig":
-        config.validate_keys(["build-uid", "build-gid", "build-os", "build-arch"])
+        config.validate_keys(["build-uid", "build-gid", "build-os", "build-arch", "remote-apis-socket"])
 
         build_os: str
         build_arch: str
@@ -132,4 +143,17 @@ def new_from_node(cls, config: "MappingNode[Node]", *, platform: Optional[Platfo
         build_uid = config.get_int("build-uid", None)
         build_gid = config.get_int("build-gid", None)
 
-        return cls(build_os=build_os, build_arch=build_arch, build_uid=build_uid, build_gid=build_gid)
+        remote_apis_socket = config.get_mapping("remote-apis-socket", default=None)
+        if remote_apis_socket:
+            remote_apis_socket.validate_keys(["path"])
+            remote_apis_socket_path = remote_apis_socket.get_str("path")
+        else:
+            remote_apis_socket_path = None
+
+        return cls(
+            build_os=build_os,
+            build_arch=build_arch,
+            build_uid=build_uid,
+            build_gid=build_gid,
+            remote_apis_socket_path=remote_apis_socket_path,
+        )

src/buildstream/sandbox/_sandboxbuildboxrun.py

@@ -83,12 +83,19 @@ def check_sandbox_config(cls, config):
         if config.build_gid is not None and "platform:unixGID" not in cls._capabilities:
             raise SandboxUnavailableError("Configuring sandbox GID is not supported by buildbox-run.")
 
+        if config.remote_apis_socket_path is not None and "platform:remoteApisSocketPath" not in cls._capabilities:
+            raise SandboxUnavailableError("Configuring Remote APIs socket path is not supported by buildbox-run.")
+
     def _execute_action(self, action, flags):
         stdout, stderr = self._get_output()
 
         context = self._get_context()
         cascache = context.get_cascache()
         casd = cascache.get_casd()
+        config = self._get_config()
+
+        if config.remote_apis_socket_path and context.remote_cache_spec:
+            raise SandboxError("'remote-apis-socket' is not currently supported with 'storage-service'.")
 
         with utils._tempnamedfile() as action_file, utils._tempnamedfile() as result_file:
             action_file.write(action.SerializeToString())

src/buildstream/sandbox/_sandboxreapi.py

@@ -131,6 +131,9 @@ def _create_platform(self, flags):
         if flags & _SandboxFlags.NETWORK_ENABLED:
             platform_dict["network"] = "on"
 
+        if config.remote_apis_socket_path:
+            platform_dict["remoteApisSocketPath"] = config.remote_apis_socket_path.lstrip(os.path.sep)
+
         # Create Platform message with properties sorted by name in code point order
         platform = remote_execution_pb2.Platform()
         for key, value in sorted(platform_dict.items()):

tests/integration/project/elements/sandbox/remote-apis-socket.bst

@@ -0,0 +1,13 @@
+kind: manual
+
+depends:
+  - filename: base.bst
+    type: build
+
+sandbox:
+  remote-apis-socket:
+    path: /tmp/reapi.sock
+
+config:
+  build-commands:
+    - test -S /tmp/reapi.sock

tests/integration/sandbox.py

@@ -48,3 +48,14 @@ def test_build_arch(cli, datafiles):
 
     result = cli.run(project=project, args=["build", element_name])
     assert result.exit_code == 0
+
+
+# Test that the REAPI socket is created in the sandbox.
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+@pytest.mark.datafiles(DATA_DIR)
+def test_remote_apis_socket(cli, datafiles):
+    project = str(datafiles)
+    element_name = "sandbox/remote-apis-socket.bst"
+
+    result = cli.run(project=project, args=["build", element_name])
+    assert result.exit_code == 0