tests: Drop internal-testing-api, move to tests-integration

Previous work started moving our tests into an external binary;
this is just cleaner because it can test things how a user
would test.  Also, we started using `libtest-mimic` to
have a "real" test scaffolding that e.g. allows selecting
individual tests to run, etc.

Complete the picture here by moving the remaining bits into
the tests-integration binary.  We now run the `tests-integration`
binary in two ways in e.g. Github Actions:

- It's compiled directly on the Ubuntu runner, and orchestrates
  things itself
- It's built in our default container image (Fedora) but as an external
  `/usr/bin/bootc-integration-tests` binary

Also while we're here, drop the kola tests.

Signed-off-by: Colin Walters <[email protected]>

Author: cgwalters

.github/workflows/ci.yml

@@ -57,9 +57,7 @@ jobs:
       - name: Build container (fedora)
         run: sudo podman build --build-arg=base=quay.io/fedora/fedora-bootc:40 -t localhost/bootc -f hack/Containerfile . 
       - name: Container integration
-        run: sudo podman run --rm localhost/bootc bootc internal-tests run-container-integration
-      - name: Privileged tests
-        run:  sudo podman run --rm --privileged -v /run/systemd:/run/systemd -v /:/run/host --pid=host localhost/bootc bootc internal-tests run-privileged-integration
+        run: sudo podman run --rm localhost/bootc bootc-integration-tests container
   cargo-deny:
     runs-on: ubuntu-latest
     steps:
@@ -78,14 +76,22 @@ jobs:
         uses: actions/checkout@v4
       - name: Ensure host skopeo is disabled
         run: sudo rm -f /bin/skopeo /usr/bin/skopeo
+      - name: Free up disk space on runner
+        run: sudo ./ci/clean-gha-runner.sh
       - name: Integration tests
         run: |
           set -xeu
           sudo podman build -t localhost/bootc -f hack/Containerfile .
+          export CARGO_INCREMENTAL=0  # because we aren't caching the test runner bits
+          cargo build --release -p tests-integration
+          df -h /
+          sudo install -m 0755 target/release/tests-integration /usr/bin/bootc-integration-tests
+          rm target -rf
+          df -h /
           # Nondestructive but privileged tests
-          cargo run -p tests-integration host-privileged localhost/bootc
+          sudo bootc-integration-tests host-privileged localhost/bootc
           # Finally the install-alongside suite
-          cargo run -p tests-integration install-alongside localhost/bootc
+          sudo bootc-integration-tests install-alongside localhost/bootc
   docs:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'documentation') }}
     runs-on: ubuntu-latest

Cargo.lock

@@ -3,9 +3,6 @@ prefix ?= /usr
 all:
 	cargo build --release
 
-all-test:
-	cargo build --release --all-features
-
 install:
 	install -D -m 0755 -t $(DESTDIR)$(prefix)/bin target/release/bootc
 	install -d -m 0755 $(DESTDIR)$(prefix)/lib/systemd/system-generators/
@@ -22,11 +19,14 @@ install:
 	  done
 	install -D -m 0644 -t $(DESTDIR)/$(prefix)/lib/systemd/system systemd/*.service systemd/*.timer
 
+install-with-tests: install
+	install -D -m 0755 target/release/tests-integration $(DESTDIR)$(prefix)/bin/bootc-integration-tests 
+
 bin-archive: all
 	$(MAKE) install DESTDIR=tmp-install && tar --zstd -C tmp-install -cf target/bootc.tar.zst . && rm tmp-install -rf
 
-test-bin-archive: all-test
-	$(MAKE) install DESTDIR=tmp-install && tar --zstd -C tmp-install -cf target/bootc.tar.zst . && rm tmp-install -rf
+test-bin-archive: all
+	$(MAKE) install-with-tests DESTDIR=tmp-install && tar --zstd -C tmp-install -cf target/bootc.tar.zst . && rm tmp-install -rf
 
 install-kola-tests:
 	install -D -t $(DESTDIR)$(prefix)/lib/coreos-assembler/tests/kola/bootc tests/kolainst/*

Makefile

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -xeuo pipefail
+df -h
+docker image prune --all --force > /dev/null
+rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android
+apt-get remove -y '^aspnetcore-.*' > /dev/null
+apt-get remove -y '^dotnet-.*' > /dev/null
+apt-get remove -y '^llvm-.*' > /dev/null
+apt-get remove -y 'php.*' > /dev/null
+apt-get remove -y '^mongodb-.*' > /dev/null
+apt-get remove -y '^mysql-.*' > /dev/null1
+apt-get remove -y azure-cli google-chrome-stable firefox mono-devel >/dev/null
+df -h

ci/clean-gha-runner.sh

@@ -54,5 +54,4 @@ default = ["install"]
 install = []
 # Implementation detail of man page generation.
 docgen = ["clap_mangen"]
-# This feature should only be enabled in CI environments.
-internal-testing-api = ["xshell"]
+

lib/Cargo.toml

@@ -161,28 +161,6 @@ impl InternalsOpts {
     const GENERATOR_BIN: &'static str = "bootc-systemd-generator";
 }
 
-/// Options for internal testing
-#[derive(Debug, clap::Subcommand, PartialEq, Eq)]
-pub(crate) enum TestingOpts {
-    /// Execute integration tests that require a privileged container
-    RunPrivilegedIntegration {},
-    /// Execute integration tests that target a not-privileged ostree container
-    RunContainerIntegration {},
-    /// Block device setup for testing
-    PrepTestInstallFilesystem { blockdev: Utf8PathBuf },
-    /// e2e test of install to-filesystem
-    TestInstallFilesystem {
-        image: String,
-        blockdev: Utf8PathBuf,
-    },
-    #[clap(name = "verify-selinux")]
-    VerifySELinux {
-        root: String,
-        #[clap(long)]
-        warn: bool,
-    },
-}
-
 /// Deploy and transactionally in-place with bootable container images.
 ///
 /// The `bootc` project currently uses ostree-containers as a backend
@@ -302,10 +280,6 @@ pub(crate) enum Opt {
     #[clap(subcommand)]
     #[clap(hide = true)]
     Internals(InternalsOpts),
-    /// Internal integration testing helpers.
-    #[clap(hide(true), subcommand)]
-    #[cfg(feature = "internal-testing-api")]
-    InternalTests(TestingOpts),
     #[clap(hide(true))]
     #[cfg(feature = "docgen")]
     Man(ManOpts),
@@ -689,8 +663,6 @@ async fn run_from_opt(opt: Opt) -> Result<()> {
             }
             InternalsOpts::FixupEtcFstab => crate::deploy::fixup_etc_fstab(&root),
         },
-        #[cfg(feature = "internal-testing-api")]
-        Opt::InternalTests(opts) => crate::privtests::run(opts).await,
         #[cfg(feature = "docgen")]
         Opt::Man(manopts) => crate::docgen::generate_manpages(&manopts.directory),
     }

lib/src/cli.rs

@@ -29,9 +29,6 @@ mod status;
 mod task;
 mod utils;
 
-#[cfg(feature = "internal-testing-api")]
-mod privtests;
-
 #[cfg(feature = "install")]
 mod blockdev;
 #[cfg(feature = "install")]

lib/src/lib.rs

@@ -17,5 +17,8 @@ cap-std-ext = "4"
 clap = { version= "4.5.4", features = ["derive","cargo"] }
 fn-error-context = "0.2.1"
 libtest-mimic = "0.7.3"
+rustix = { "version" = "0.38.34", features = ["thread", "fs", "system", "process"] }
+serde = { features = ["derive"], version = "1.0.199" }
+serde_json = "1.0.116"
 tempfile = "3.10.1"
 xshell = { version = "0.2.6" }