Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Adds an --rbe option to tools/gn that works on Linux hosts #45271

Merged
merged 1 commit into from
Aug 31, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions DEPS
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ vars = {
# https://chrome-infra-packages.appspot.com/p/fuchsia/third_party/goma/client
'goma_version': ' git_revision:41b3bcb64014144a844153fd5588c36411fffb56',

'reclient_version': 'git_revision:81e819b39d4743462857cc55430d898b9fcca1af',

'gcloud_version': 'version:[email protected]',

# When updating the Dart revision, ensure that all entries that are
# dependencies of Dart are also updated to match the entries in the
# Dart SDK's DEPS file for that revision of Dart. The DEPS file for
Expand Down Expand Up @@ -880,6 +884,30 @@ deps = {
'dep_type': 'cipd',
},

# reclient.
'src/buildtools/linux-x64/reclient': {
'packages': [
{
'package': 'infra/rbe/client/${{platform}}',
'version': Var('reclient_version'),
}
],
'condition': 'host_os == "linux" and host_cpu == "x64"',
'dep_type': 'cipd',
},

# gcloud
'src/buildtools/linux-x64/gcloud': {
'packages': [
{
'package': 'infra/3pp/tools/gcloud/${{platform}}',
'version': Var('gcloud_version'),
}
],
'condition': 'host_os == "linux" and host_cpu == "x64"',
'dep_type': 'cipd',
},

# Get the SDK from https://chrome-infra-packages.appspot.com/p/fuchsia/sdk/core at the 'latest' tag
# Get the toolchain from https://chrome-infra-packages.appspot.com/p/fuchsia/clang at the 'goma' tag
'src/fuchsia/sdk/mac': {
Expand Down
98 changes: 98 additions & 0 deletions tools/gn
Original file line number Diff line number Diff line change
Expand Up @@ -214,8 +214,69 @@ def buildtools_dir():
return '%s-%s' % (host_os, host_cpu)


def setup_rbe(args):
rbe_gn_args = {}
# RBE is default-off. If it is not asked for, then silently keep all default
# flag values.
if not args.rbe:
return rbe_gn_args

if get_host_os() not in ['linux']:
print(
'The --rbe flag has no effect. RBE is currently only supported on Linux.'
)
return rbe_gn_args

rbe_gn_args['use_rbe'] = True

# When running in CI, the recipes use their own rbe install, and take
# care of starting and stopping the compiler proxy.
running_on_luci = os.environ.get('LUCI_CONTEXT') is not None

# Bootstrap reproxy if not running in CI.
if not running_on_luci:
cipd_reclient_dir = os.path.join(
SRC_ROOT,
'buildtools',
buildtools_dir(),
'reclient',
)
bootstrap_path = os.path.join(cipd_reclient_dir, 'bootstrap')
reproxy_path = os.path.join(cipd_reclient_dir, 'reproxy')
rbe_cfg_path = os.path.join(SRC_ROOT, 'build', 'rbe.cfg')
bootstrap_cmd = [
bootstrap_path,
'--re_proxy=' + reproxy_path,
'--automatic_auth=true',
'--cfg=' + rbe_cfg_path,
]
try:
subprocess.call(bootstrap_cmd, cwd=SRC_ROOT)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From LUCI side, we expect to have a cleanup/shutdown step after task finishes. You may want to mention somewhere that users are expected to cleanup/shutdown the bootstrap later.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There isn't a good spot to put that in the local dev workflow, and the command is too long to ask folks to run by hand, so that will need some more thought.

except subprocess.CalledProcessError as exc:
print('Failed to boostrap reproxy: ', exc.returncode, exc.output)
return {}

if args.rbe_server_address:
rbe_gn_args['rbe_server_address'] = args.rbe_server_address
if args.rbe_exec_strategy:
rbe_gn_args['rbe_exec_strategy'] = args.rbe_exec_strategy
if args.rbe_dial_timeout:
rbe_gn_args['rbe_dial_timeout'] = args.rbe_dial_timeout
if args.rbe_platform:
rbe_gn_args['rbe_platform'] = args.rbe_platform
if args.rbe_dir:
rbe_gn_args['rbe_dir'] = args.rbe_dir

return rbe_gn_args


def setup_goma(args):
goma_gn_args = {}
# If RBE is requested, don't try to use goma.
if args.rbe:
goma_gn_args['use_goma'] = False
goma_gn_args['goma_dir'] = None
return goma_gn_args

# args.goma has three states, True (--goma), False (--no-goma), and
# None (default). In True mode, we force GOMA to be used (and fail
Expand Down Expand Up @@ -290,6 +351,8 @@ def to_gn_args(args):

gn_args.update(setup_goma(args))

gn_args.update(setup_rbe(args))

# If building for WASM, set the GN args using 'to_gn_wasm_args' as most
# of the Flutter SDK specific arguments are unused.
if args.target_os == 'wasm' or args.web:
Expand Down Expand Up @@ -913,6 +976,41 @@ def parse_args(args):
help='Do not build the host-side development artifacts.'
)

parser.add_argument('--rbe', default=None, action='store_true')
parser.add_argument('--no-rbe', dest='rbe', action='store_false')
parser.add_argument(
'--rbe-server-address',
default=None,
type=str,
help='The reproxy serveraddress'
)
parser.add_argument(
'--rbe-exec-strategy',
default=None,
type=str,
help='The RBE execution strategy.',
choices=['local', 'remote', 'remote_local_fallback', 'racing']
)
parser.add_argument(
'--rbe-dial-timeout',
default=None,
type=str,
help='The timeout for connecting to the local reproxy server.',
)
parser.add_argument(
'--rbe-platform',
default=None,
type=str,
help='The RBE "platform" string. This is used to identify remote platform '
'settings like the docker image to use to run the command.'
)
parser.add_argument(
'--rbe-dir',
default=None,
type=str,
help='The location of the reclient binaries.'
)

parser.add_argument('--goma', default=None, action='store_true')
parser.add_argument('--no-goma', dest='goma', action='store_false')
parser.add_argument(
Expand Down