Skip to content

Replace fvdl with ffx, allow test without install #112719

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 17, 2023
+100 −156
Merged

Replace fvdl with ffx, allow test without install #112719

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4c6fd75
Replace fvdl with ffx, allow test without install
Jun 16, 2023
08d3065
Fix --rust-build flag in docs
tmandry Jun 16, 2023
12c6f1d
Fix edit fail
tmandry Jun 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
237 changes: 92 additions & 145 deletions src/ci/docker/scripts/fuchsia-test-runner.py
View file
Original file line number Diff line number Diff line change
@@ -25,13 +25,9 @@

@dataclass
class TestEnvironment:
rust_dir: str
rust_build_dir: str
sdk_dir: str
target: str
package_server_pid: Optional[int] = None
emu_addr: Optional[str] = None
libstd_name: Optional[str] = None
libtest_name: Optional[str] = None
verbose: bool = False

@staticmethod
@@ -57,7 +53,7 @@ def env_file_path(cls):
@classmethod
def from_args(cls, args):
return cls(
os.path.abspath(args.rust),
os.path.abspath(args.rust_build),
os.path.abspath(args.sdk),
args.target,
verbose=args.verbose,
@@ -68,32 +64,16 @@ def read_from_file(cls):
with open(cls.env_file_path(), encoding="utf-8") as f:
test_env = json.loads(f.read())
return cls(
test_env["rust_dir"],
test_env["rust_build_dir"],
test_env["sdk_dir"],
test_env["target"],
libstd_name=test_env["libstd_name"],
libtest_name=test_env["libtest_name"],
emu_addr=test_env["emu_addr"],
package_server_pid=test_env["package_server_pid"],
verbose=test_env["verbose"],
)

def write_to_file(self):
with open(self.env_file_path(), "w", encoding="utf-8") as f:
f.write(json.dumps(self.__dict__))

def ssh_dir(self):
return os.path.join(self.tmp_dir(), "ssh")

def ssh_keyfile_path(self):
return os.path.join(self.ssh_dir(), "fuchsia_ed25519")

def ssh_authfile_path(self):
return os.path.join(self.ssh_dir(), "fuchsia_authorized_keys")

def vdl_output_path(self):
return os.path.join(self.tmp_dir(), "vdl_output")

def package_server_log_path(self):
return os.path.join(self.tmp_dir(), "package_server_log")

@@ -113,7 +93,9 @@ def repo_dir(self):

def libs_dir(self):
return os.path.join(
self.rust_dir,
self.rust_build_dir,
"host",
"stage2",
"lib",
)

@@ -212,21 +194,19 @@ def start_ffx_isolation(self):
# Set configs
configs = {
"log.enabled": "true",
"ssh.pub": self.ssh_authfile_path(),
"ssh.priv": self.ssh_keyfile_path(),
"test.is_isolated": "true",
"test.experimental_structured_output": "true",
}
for key, value in configs.items():
subprocess.check_call(
[
self.tool_path("ffx"),
ffx_path,
"config",
"set",
key,
value,
],
env=self.ffx_cmd_env(),
env=ffx_env,
stdout=self.subprocess_output(),
stderr=self.subprocess_output(),
)
@@ -248,6 +228,7 @@ def stop_ffx_isolation(self):
self.tool_path("ffx"),
"daemon",
"stop",
"-w",
],
env=self.ffx_cmd_env(),
stdout=self.subprocess_output(),
@@ -275,87 +256,62 @@ def start(self):
elif len(os.listdir(self.tmp_dir())) != 0:
raise Exception(f"Temp directory is not clean (in {self.tmp_dir()})")

os.mkdir(self.ssh_dir())
os.mkdir(self.output_dir())

# Find libstd and libtest
libstd_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libstd-*.so"))
libtest_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libtest-*.so"))

if not libstd_paths:
raise Exception(f"Failed to locate libstd (in {self.rustlibs_dir()})")

if not libtest_paths:
raise Exception(f"Failed to locate libtest (in {self.rustlibs_dir()})")
ffx_path = self.tool_path("ffx")
ffx_env = self.ffx_cmd_env()

self.libstd_name = os.path.basename(libstd_paths[0])
self.libtest_name = os.path.basename(libtest_paths[0])
# Start ffx isolation
self.log_info("Starting ffx isolation...")
self.start_ffx_isolation()

# Generate SSH keys for the emulator to use
self.log_info("Generating SSH keys...")
# Stop any running emulators (there shouldn't be any)
subprocess.check_call(
[
"ssh-keygen",
"-N",
"",
"-t",
"ed25519",
"-f",
self.ssh_keyfile_path(),
"-C",
"Generated by fuchsia-test-runner.py",
ffx_path,
"emu",
"stop",
"--all",
],
env=ffx_env,
stdout=self.subprocess_output(),
stderr=self.subprocess_output(),
)
authfile_contents = subprocess.check_output(

# Start emulator
self.log_info("Starting emulator...")
product_bundle = "terminal.qemu-" + self.triple_to_arch(self.target)
subprocess.check_call(
[
"ssh-keygen",
"-y",
"-f",
self.ssh_keyfile_path(),
ffx_path,
"product-bundle",
"get",
product_bundle,
],
env=ffx_env,
stdout=self.subprocess_output(),
stderr=self.subprocess_output(),
)
with open(self.ssh_authfile_path(), "wb") as authfile:
authfile.write(authfile_contents)

# Start ffx isolation
self.log_info("Starting ffx isolation...")
self.start_ffx_isolation()

# Start emulator (this will generate the vdl output)
self.log_info("Starting emulator...")
# FIXME: condition --accel hyper on target arch matching host arch
subprocess.check_call(
[
self.tool_path("fvdl"),
"--sdk",
ffx_path,
"emu",
"start",
"--tuntap",
product_bundle,
"--headless",
"--nointeractive",
"--ssh",
self.ssh_dir(),
"--vdl-output",
self.vdl_output_path(),
"--emulator-log",
"--log",
self.emulator_log_path(),
"--image-name",
"qemu-" + self.triple_to_arch(self.target),
"--net",
"tap",
"--accel",
"hyper",
],
env=ffx_env,
stdout=self.subprocess_output(),
stderr=self.subprocess_output(),
)

# Parse vdl output for relevant information
with open(self.vdl_output_path(), encoding="utf-8") as f:
vdl_content = f.read()
matches = re.search(
r'network_address:\s+"\[([0-9a-f]{1,4}:(:[0-9a-f]{1,4}){4}%qemu)\]"',
vdl_content,
)
self.emu_addr = matches.group(1)

# Create new package repo
self.log_info("Creating package repo...")
subprocess.check_call(
@@ -369,55 +325,40 @@ def start(self):
stderr=self.subprocess_output(),
)

# Start package server
self.log_info("Starting package server...")
with open(
self.package_server_log_path(), "w", encoding="utf-8"
) as package_server_log:
# We want this to be a long-running process that persists after the script finishes
# pylint: disable=consider-using-with
self.package_server_pid = subprocess.Popen(
[
self.tool_path("pm"),
"serve",
"-vt",
"-repo",
self.repo_dir(),
"-l",
":8084",
],
stdout=package_server_log,
stderr=package_server_log,
).pid

# Register package server with emulator
self.log_info("Registering package server...")
ssh_client = subprocess.check_output(
# Add repo
subprocess.check_call(
[
"ssh",
"-i",
self.ssh_keyfile_path(),
"-o",
"StrictHostKeyChecking=accept-new",
self.emu_addr,
"-f",
"echo $SSH_CLIENT",
ffx_path,
"repository",
"add-from-pm",
self.repo_dir(),
"--repository",
self.TEST_REPO_NAME,
],
text=True,
env=ffx_env,
stdout=self.subprocess_output(),
stderr=self.subprocess_output(),
)
repo_addr = ssh_client.split()[0].replace("%", "%25")
repo_url = f"http://[{repo_addr}]:8084/config.json"

# Start repository server
subprocess.check_call(
[ffx_path, "repository", "server", "start", "--address", "[::]:0"],
env=ffx_env,
stdout=self.subprocess_output(),
stderr=self.subprocess_output(),
)

# Register with newly-started emulator
subprocess.check_call(
[
"ssh",
"-i",
self.ssh_keyfile_path(),
"-o",
"StrictHostKeyChecking=accept-new",
self.emu_addr,
"-f",
f"pkgctl repo add url -f 1 -n {self.TEST_REPO_NAME} {repo_url}",
ffx_path,
"target",
"repository",
"register",
"--repository",
self.TEST_REPO_NAME,
],
env=ffx_env,
stdout=self.subprocess_output(),
stderr=self.subprocess_output(),
)
@@ -471,8 +412,8 @@ def start(self):
meta/package={package_dir}/meta/package
meta/{package_name}.cm={package_dir}/meta/{package_name}.cm
bin/{exe_name}={bin_path}
lib/{libstd_name}={rust_dir}/lib/rustlib/{rustlib_dir}/lib/{libstd_name}
lib/{libtest_name}={rust_dir}/lib/rustlib/{rustlib_dir}/lib/{libtest_name}
lib/{libstd_name}={libstd_path}
lib/{libtest_name}={libtest_path}
lib/ld.so.1={sdk_dir}/arch/{target_arch}/sysroot/dist/lib/ld.so.1
lib/libfdio.so={sdk_dir}/arch/{target_arch}/dist/libfdio.so
"""
@@ -502,6 +443,16 @@ def run(self, args):

bin_path = os.path.abspath(args.bin_path)

# Find libstd and libtest
libstd_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libstd-*.so"))
libtest_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libtest-*.so"))

if not libstd_paths:
raise Exception(f"Failed to locate libstd (in {self.rustlibs_dir()})")

if not libtest_paths:
raise Exception(f"Failed to locate libtest (in {self.rustlibs_dir()})")

# Build a unique, deterministic name for the test using the name of the
# binary and the last 6 hex digits of the hash of the full path
def path_checksum(path):
@@ -604,11 +555,12 @@ def log(msg):
exe_name=exe_name,
package_dir=package_dir,
package_name=package_name,
rust_dir=self.rust_dir,
rustlib_dir=self.target,
target=self.target,
sdk_dir=self.sdk_dir,
libstd_name=self.libstd_name,
libtest_name=self.libtest_name,
libstd_name=os.path.basename(libstd_paths[0]),
libtest_name=os.path.basename(libtest_paths[0]),
libstd_path=libstd_paths[0],
libtest_path=libtest_paths[0],
target_arch=self.triple_to_arch(self.target),
)
)
@@ -779,20 +731,15 @@ def stop(self):
else:
self.log_debug("No ffx daemon log found")

# Stop package server
self.log_info("Stopping package server...")
os.kill(self.package_server_pid, signal.SIGTERM)

# Shut down the emulator
self.log_info("Stopping emulator...")
subprocess.check_call(
[
self.tool_path("fvdl"),
"--sdk",
"kill",
"--launched-proto",
self.vdl_output_path(),
self.tool_path("ffx"),
"emu",
"stop",
],
env=self.ffx_cmd_env(),
stdout=self.subprocess_output(),
stderr=self.subprocess_output(),
)
@@ -969,8 +916,8 @@ def print_help(args):
"start", help="initializes the testing environment"
)
start_parser.add_argument(
"--rust",
help="the directory of the installed Rust compiler for Fuchsia",
"--rust-build",
help="the current compiler build directory (`$RUST_SRC/build` by default)",
required=True,
)
start_parser.add_argument(
19 changes: 8 additions & 11 deletions src/doc/rustc/src/platform-support/fuchsia.md
View file
Original file line number Diff line number Diff line change
@@ -681,12 +681,9 @@ local Rust source checkout:
cd ${RUST_SRC_PATH}
```

To run the Rust test suite on an emulated Fuchsia device, you must install the
Rust compiler locally. See "[Targeting Fuchsia with a compiler built from source](#targeting-fuchsia-with-a-compiler-built-from-source)"
for the steps to build locally.

You'll also need to download a copy of the Fuchsia SDK. The current minimum
supported SDK version is [10.20221207.2.89][minimum_supported_sdk_version].
To run the Rust test suite on an emulated Fuchsia device, you'll also need to
download a copy of the Fuchsia SDK. The current minimum supported SDK version is
[10.20221207.2.89][minimum_supported_sdk_version].

[minimum_supported_sdk_version]: https://chrome-infra-packages.appspot.com/p/fuchsia/sdk/core/linux-amd64/+/version:10.20221207.2.89

@@ -695,13 +692,13 @@ Fuchsia's test runner interacts with the Fuchsia emulator and is located at
test environment with:

```sh
src/ci/docker/scripts/fuchsia-test-runner.py start
--rust ${RUST_SRC_PATH}/install
--sdk ${SDK_PATH}
--target {x86_64-unknown-fuchsia|aarch64-unknown-fuchsia}
src/ci/docker/scripts/fuchsia-test-runner.py start \
--rust-build ${RUST_SRC_PATH}/build \
--sdk ${SDK_PATH} \
--target {x86_64-unknown-fuchsia|aarch64-unknown-fuchsia} \
```

Where `${RUST_SRC_PATH}/install` is the `prefix` set in `config.toml` and
Where `${RUST_SRC_PATH}/build` is the `build-dir` set in `config.toml` and
`${SDK_PATH}` is the path to the downloaded and unzipped SDK.

Once our environment is started, we can run our tests using `x.py` as usual. The