WIP: Measurement Corpus (take 2)

Cargo.toml

@@ -647,6 +647,7 @@ rand_distr = "0.4.3"
 rand_seeder = "0.3.0"
 range-requests = { path = "range-requests" }
 ratatui = "0.29.0"
+rats-corim = { git = "https://github.com/oxidecomputer/rats-corim.git", branch = "main" }
 rayon = "1.10"
 rcgen = "0.12.1"
 reconfigurator-cli = { path = "dev-tools/reconfigurator-cli" }
@@ -747,10 +748,10 @@ toml_edit = "0.22.27"
 tough = { version = "0.20.0", features = [ "http" ] }
 transceiver-controller = { git = "https://github.com/oxidecomputer/transceiver-control", features = [ "api-traits" ] }
 trybuild = "1.0.106"
-tufaceous = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main" }
-tufaceous-artifact = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main", features = ["proptest", "schemars"] }
-tufaceous-brand-metadata = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main" }
-tufaceous-lib = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main" }
+tufaceous = { git = "https://github.com/oxidecomputer/tufaceous", branch = "measurement_corpus_take_2" }
+tufaceous-artifact = { git = "https://github.com/oxidecomputer/tufaceous", branch = "measurement_corpus_take_2", features = ["proptest", "schemars"] }
+tufaceous-brand-metadata = { git = "https://github.com/oxidecomputer/tufaceous", branch = "measurement_corpus_take_2" }
+tufaceous-lib = { git = "https://github.com/oxidecomputer/tufaceous", branch = "measurement_corpus_take_2" }
 tui-tree-widget = "0.23.1"
 typed-rng = { path = "typed-rng" }
 typify = "0.3.0"

dev-tools/releng/Cargo.toml

@@ -17,6 +17,7 @@ hex.workspace = true
 omicron-pins.workspace = true
 omicron-workspace-hack.workspace = true
 omicron-zone-package.workspace = true
+rats-corim.workspace = true
 reqwest.workspace = true
 semver.workspace = true
 serde.workspace = true

dev-tools/releng/src/hubris.rs

@@ -38,6 +38,19 @@ pub(crate) async fn fetch_hubris_artifacts(
 
     fs::create_dir_all(&output_dir).await?;
 
+    // We need to remove our old downloaded corpus to make sure nothing else
+    // gets added to the repo unexpectedly. This should only really be a
+    // issue with local builds
+    if std::fs::exists(&output_dir.join("measurement_corpus"))
+        .context("failed to check `measurement_corpus`")?
+    {
+        std::fs::remove_dir_all(&output_dir.join("measurement_corpus"))
+            .context("failed to remove `measurement_corpus")?;
+    }
+    fs::create_dir_all(&output_dir.join("measurement_corpus"))
+        .await
+        .context("Failed to create `measurement_corpus`")?;
+
     // This could be parallelized with FuturesUnordered but in practice this
     // takes less time than OS builds.
 
@@ -106,6 +119,22 @@ pub(crate) async fn fetch_hubris_artifacts(
                     }
                 }
             }
+            if let Some(corpus) = hash_manifest.corpus {
+                let hash = match corpus {
+                    Source::File(file) => file.hash,
+                    _ => anyhow::bail!(
+                        "Unexpected file type: should be a single file, not an RoT"
+                    ),
+                };
+                let data =
+                    fetch_hash(&logger, base_url, &client, &hash).await?;
+                fs::write(
+                    output_dir.join("measurement_corpus").join(hash),
+                    data,
+                )
+                .await
+                .context("failed to write file {hash}")?;
+            }
         }
     }
 
@@ -160,6 +189,9 @@ async fn fetch_hash(
 struct Manifest {
     #[serde(rename = "artifact")]
     artifacts: HashMap<KnownArtifactKind, Vec<Artifact>>,
+    // Add a default for backwards compatibility
+    #[serde(rename = "measurement_corpus")]
+    corpus: Option<Source>,
 }
 
 #[derive(Deserialize)]

dev-tools/releng/src/tuf.rs

@@ -76,6 +76,46 @@ pub(crate) async fn build_tuf_repo(
         }
     }
 
+    let mut measurement_corpus = vec![];
+
+    for entry in std::fs::read_dir(
+        output_dir.join("hubris-staging").join("measurement_corpus"),
+    )
+    .context("failed to read `hubris-staging/measurement_corpus")?
+    {
+        let entry = entry?;
+
+        let corim = rats_corim::Corim::from_file(entry.path())?;
+
+        measurement_corpus.push(DeserializedArtifactData {
+            name: format!("staging-{}", corim.id),
+            version: ArtifactVersion::new(corim.get_version()?.clone())?,
+            source: DeserializedArtifactSource::File {
+                path: Utf8PathBuf::from_path_buf(entry.path()).unwrap(),
+            },
+        });
+    }
+
+    for entry in std::fs::read_dir(
+        output_dir.join("hubris-production").join("measurement_corpus"),
+    )
+    .context("failed to read `hubris-production/measurement_corpus")?
+    {
+        let entry = entry?;
+        let corim = rats_corim::Corim::from_file(entry.path())?;
+        measurement_corpus.push(DeserializedArtifactData {
+            name: format!("production-{}", corim.id),
+            version: ArtifactVersion::new(corim.get_version()?.clone())?,
+            source: DeserializedArtifactSource::File {
+                path: Utf8PathBuf::from_path_buf(entry.path()).unwrap(),
+            },
+        });
+    }
+
+    manifest
+        .artifacts
+        .insert(KnownArtifactKind::MeasurementCorpus, measurement_corpus);
+
     // Add the OS images.
     manifest.artifacts.insert(
         KnownArtifactKind::Host,
@@ -115,6 +155,7 @@ pub(crate) async fn build_tuf_repo(
                 .join(format!("{}.tar.gz", package)),
         });
     }
+
     manifest.artifacts.insert(
         KnownArtifactKind::ControlPlane,
         vec![DeserializedArtifactData {

installinator/src/dispatch.rs

@@ -279,6 +279,9 @@ impl InstallOpts {
                                         control_plane_hash =
                                             Some(artifact.hash);
                                     }
+                                    InstallinatorArtifactKind::MeasurementCorpus => {
+    // Skip doing anything with this for the moment
+},
                                 }
                             }
 

nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs

@@ -144,6 +144,7 @@ pub fn try_make_update_rot_bootloader(
                     | KnownArtifactKind::Trampoline
                     | KnownArtifactKind::ControlPlane
                     | KnownArtifactKind::Zone
+                    | KnownArtifactKind::MeasurementCorpus
                     | KnownArtifactKind::PscRot
                     | KnownArtifactKind::SwitchRot
                     | KnownArtifactKind::GimletSp

nexus/reconfigurator/planning/src/mgs_updates/sp.rs

@@ -121,6 +121,7 @@ pub fn try_make_update_sp(
                     | KnownArtifactKind::InstallinatorDocument
                     | KnownArtifactKind::ControlPlane
                     | KnownArtifactKind::Zone
+                    | KnownArtifactKind::MeasurementCorpus
                     | KnownArtifactKind::PscRot
                     | KnownArtifactKind::SwitchRot
                     | KnownArtifactKind::GimletRotBootloader

update-common/manifests/fake-non-semver.toml

@@ -94,3 +94,8 @@ source = { kind = "fake", size = "1MiB" }
 name = "fake-switch-rot-bootloader"
 version = "non-semver-2"
 source = { kind = "fake", size = "1MiB" }
+
+[[artifact.measurement_corpus]]
+name = "fake-corpus"
+version = "2.0.0"
+source = { kind = "fake", size = "1MiB" }

update-common/manifests/fake.toml

@@ -93,4 +93,7 @@ name = "fake-switch-rot-bootloader"
 version = "1.0.0"
 source = { kind = "fake", size = "1MiB" }
 
-
+[[artifact.measurement_corpus]]
+name = "fake-corpus"
+version = "1.0.0"
+source = { kind = "fake", size = "1MiB" }

update-common/src/artifacts/update_plan.rs

@@ -275,7 +275,7 @@ impl<'a> UpdatePlanBuilder<'a> {
                 )
                 .await
             }
-            KnownArtifactKind::Zone => {
+            KnownArtifactKind::Zone | KnownArtifactKind::MeasurementCorpus => {
                 // We don't currently support repos with already split-out
                 // zones.
                 self.add_unknown_artifact(artifact_id, artifact_hash, stream)
@@ -302,6 +302,7 @@ impl<'a> UpdatePlanBuilder<'a> {
             | KnownArtifactKind::InstallinatorDocument
             | KnownArtifactKind::ControlPlane
             | KnownArtifactKind::Zone
+            | KnownArtifactKind::MeasurementCorpus
             | KnownArtifactKind::PscRot
             | KnownArtifactKind::SwitchRot
             | KnownArtifactKind::GimletRotBootloader
@@ -397,6 +398,7 @@ impl<'a> UpdatePlanBuilder<'a> {
             | KnownArtifactKind::InstallinatorDocument
             | KnownArtifactKind::ControlPlane
             | KnownArtifactKind::Zone
+            | KnownArtifactKind::MeasurementCorpus
             | KnownArtifactKind::PscRot
             | KnownArtifactKind::SwitchRot
             | KnownArtifactKind::GimletSp
@@ -503,6 +505,7 @@ impl<'a> UpdatePlanBuilder<'a> {
             | KnownArtifactKind::InstallinatorDocument
             | KnownArtifactKind::ControlPlane
             | KnownArtifactKind::Zone
+            | KnownArtifactKind::MeasurementCorpus
             | KnownArtifactKind::PscSp
             | KnownArtifactKind::SwitchSp
             | KnownArtifactKind::GimletRotBootloader
@@ -1979,6 +1982,30 @@ mod tests {
                 .unwrap();
         }
 
+        // The measurement corpus can be random bytes
+        // If we do further testing we should use a fake CoRIM
+        let measurement_corpus_data = make_random_bytes();
+        let measurement_corpus_hash =
+            ArtifactHash(Sha256::digest(&measurement_corpus_data).into());
+        {
+            let kind = KnownArtifactKind::MeasurementCorpus;
+            let id = ArtifactId {
+                name: format!("{kind:?}"),
+                version: ARTIFACT_VERSION_0,
+                kind: kind.into(),
+            };
+            plan_builder
+                .add_artifact(
+                    id,
+                    measurement_corpus_hash,
+                    futures::stream::iter([Ok(Bytes::from(
+                        measurement_corpus_data,
+                    ))]),
+                )
+                .await
+                .unwrap();
+        }
+
         // For each SP image, we'll insert two artifacts: these should end up in
         // the update plan's SP image maps keyed by their "board". Normally the
         // board is read from the archive itself via hubtools; we'll inject a
@@ -2165,6 +2192,10 @@ mod tests {
                         hash_ids[0].hash
                     );
                 }
+                KnownArtifactKind::MeasurementCorpus => {
+                    assert_eq!(hash_ids.len(), 1);
+                    assert_eq!(measurement_corpus_hash, hash_ids[0].hash);
+                }
                 // These are special (we import their inner parts) and we'll
                 // check them below.
                 KnownArtifactKind::Host

workspace-hack/Cargo.toml

@@ -117,6 +117,7 @@ scopeguard = { version = "1.2.0" }
 semver = { version = "1.0.26", features = ["serde"] }
 serde = { version = "1.0.219", features = ["alloc", "derive", "rc"] }
 serde_json = { version = "1.0.142", features = ["raw_value", "unbounded_depth"] }
+serde_with = { version = "3.14.0" }
 sha1 = { version = "0.10.6", features = ["oid"] }
 sha2 = { version = "0.10.9", features = ["oid"] }
 similar = { version = "2.7.0", features = ["bytes", "inline", "unicode"] }
@@ -254,6 +255,7 @@ scopeguard = { version = "1.2.0" }
 semver = { version = "1.0.26", features = ["serde"] }
 serde = { version = "1.0.219", features = ["alloc", "derive", "rc"] }
 serde_json = { version = "1.0.142", features = ["raw_value", "unbounded_depth"] }
+serde_with = { version = "3.14.0" }
 sha1 = { version = "0.10.6", features = ["oid"] }
 sha2 = { version = "0.10.9", features = ["oid"] }
 similar = { version = "2.7.0", features = ["bytes", "inline", "unicode"] }