support for tunneled routing (#381)

Author: rcgoodfellow

.github/buildomat/jobs/oxide-vpc.sh

@@ -2,12 +2,11 @@
 #:
 #: name = "test"
 #: variety = "basic"
-#: target = "lab-2.0-opte-0.23"
+#: target = "helios-2.0"
 #: rust_toolchain = "stable"
 #: output_rules = [
 #:   "/work/*.log",
 #: ]
-#: skip_clone = true
 #:
 #: [dependencies.xde]
 #: job = "opte-xde"
@@ -22,6 +21,8 @@
 
 set -o xtrace
 
+pfexec pkg install brand/sparse opte
+
 if [[ -z $BUILDOMAT_JOB_ID ]]; then
     echo Note: if you are running this locally, you must run the xde.sh job first
     echo to have the artifacts at the expected spot.
@@ -37,6 +38,30 @@ function cleanup {
 }
 trap cleanup EXIT
 
+function get_artifact {
+    repo=$1
+    series=$2
+    commit=$3
+    name=$4
+    url=https://buildomat.eng.oxide.computer/public/file/oxidecomputer
+
+    mkdir -p download
+    pushd download
+    if [[ ! -f $name ]]; then
+        curl -fOL $url/$repo/$series/$commit/$name
+    fi
+    popd
+}
+
+banner "collect"
+get_artifact softnpu image 88f5f1334364e5580fe778c44ac0746a35927351 softnpu
+get_artifact sidecar-lite release 3fff53ae549ab1348b680845693e66b224bb5d2f libsidecar_lite.so
+get_artifact sidecar-lite release 3fff53ae549ab1348b680845693e66b224bb5d2f scadm
+
+if [[ $DOWNLOAD_ONLY -eq 1 ]]; then
+    exit 0;
+fi
+
 uname -a
 cat /etc/versions/build
 

.github/buildomat/jobs/test.sh

@@ -17,6 +17,15 @@
 #:   "=/work/xde.conf",
 #: ]
 #:
+#: [[publish]]
+#: series = "module"
+#: name = "xde"
+#: from_output = "/work/release/xde"
+#
+#: [[publish]]
+#: series = "module"
+#: name = "xde.sha256"
+#: from_output = "/work/release/xde.sha256"
 
 set -o errexit
 set -o pipefail
@@ -69,10 +78,12 @@ header "check style"
 ptime -m cargo +nightly-2023-10-23 fmt -p xde -p xde-link -- --check
 
 header "analyze"
-ptime -m cargo clippy -- --allow clippy::uninlined-format-args
+ptime -m cargo clippy -- \
+    --allow clippy::uninlined-format-args --allow clippy::bad_bit_mask
 
 pushd xde-link
-ptime -m cargo clippy -- --allow clippy::uninlined-format-args
+ptime -m cargo clippy -- \
+    --allow clippy::uninlined-format-args --allow clippy::bad_bit_mask
 popd
 
 header "build xde (debug)"

.github/buildomat/jobs/xde.sh

@@ -1,2 +1,3 @@
 *.html
 target
+download

.gitignore

@@ -60,6 +60,7 @@ version_check = "0.9"
 zerocopy = { version = "0.7", features = ["derive"] }
 zone = { git = "https://github.com/oxidecomputer/zone" }
 ztest = { git = "https://github.com/oxidecomputer/falcon", branch = "main" }
+poptrie = { git = "https://github.com/oxidecomputer/poptrie", branch = "multipath" }
 
 [profile.release]
 debug = 2

Cargo.lock

@@ -24,7 +24,7 @@ use opteadm::OpteAdm;
 use opteadm::COMMIT_COUNT;
 use oxide_vpc::api::AddRouterEntryReq;
 use oxide_vpc::api::Address;
-use oxide_vpc::api::BoundaryServices;
+use oxide_vpc::api::ClearVirt2BoundaryReq;
 use oxide_vpc::api::DhcpCfg;
 use oxide_vpc::api::ExternalIpCfg;
 use oxide_vpc::api::Filters as FirewallFilters;
@@ -42,15 +42,20 @@ use oxide_vpc::api::RouterTarget;
 use oxide_vpc::api::SNat4Cfg;
 use oxide_vpc::api::SNat6Cfg;
 use oxide_vpc::api::SetExternalIpsReq;
+use oxide_vpc::api::SetVirt2BoundaryReq;
 use oxide_vpc::api::SetVirt2PhysReq;
+use oxide_vpc::api::TunnelEndpoint;
 use oxide_vpc::api::VpcCfg;
+use oxide_vpc::engine::overlay::BOUNDARY_SERVICES_VNI;
+use oxide_vpc::engine::print::print_v2b;
 use oxide_vpc::engine::print::print_v2p;
 use std::io;
 use std::str::FromStr;
 
 /// Administer the Oxide Packet Transformation Engine (OPTE)
 #[derive(Debug, Parser)]
 #[command(version=opte_pkg_version())]
+#[allow(clippy::large_enum_variant)]
 enum Command {
     /// List all ports.
     ListPorts,
@@ -96,6 +101,9 @@ enum Command {
     /// Dump virtual to physical address mapping
     DumpV2P,
 
+    /// Dump virtual to boundary address mapping
+    DumpV2B,
+
     /// Add a firewall rule
     AddFwRule {
         #[arg(short)]
@@ -162,19 +170,6 @@ enum Command {
         #[arg(long)]
         gateway_ip: IpAddr,
 
-        /// The IP address for Boundary Services, where packets destined to
-        /// off-rack networks are sent.
-        #[arg(long)]
-        bsvc_addr: Ipv6Addr,
-
-        /// The VNI used for Boundary Services.
-        #[arg(long)]
-        bsvc_vni: Vni,
-
-        /// The MAC address for Boundary Services.
-        #[arg(long, default_value = "00:00:00:00:00:00")]
-        bsvc_mac: MacAddr,
-
         /// The VNI for the VPC to which the guest belongs.
         #[arg(long)]
         vpc_vni: Vni,
@@ -203,6 +198,12 @@ enum Command {
     /// Set a virtual-to-physical mapping
     SetV2P { vpc_ip: IpAddr, vpc_mac: MacAddr, underlay_ip: Ipv6Addr, vni: Vni },
 
+    /// Set a virtual-to-boundary mapping
+    SetV2B { prefix: IpCidr, tunnel_endpoint: Vec<Ipv6Addr> },
+
+    /// Clear a virtual-to-boundary mapping
+    ClearV2B { prefix: IpCidr, tunnel_endpoint: Vec<Ipv6Addr> },
+
     /// Add a new router entry, either IPv4 or IPv6.
     AddRouterEntry {
         /// The OPTE port to which the route is added
@@ -511,6 +512,11 @@ fn main() -> anyhow::Result<()> {
             print_v2p(&hdl.dump_v2p()?);
         }
 
+        Command::DumpV2B => {
+            let hdl = opteadm::OpteAdm::open(OpteAdm::XDE_CTL)?;
+            print_v2b(&hdl.dump_v2b()?);
+        }
+
         Command::AddFwRule { port, direction, filters, action, priority } => {
             let rule = FirewallRule {
                 direction,
@@ -541,9 +547,6 @@ fn main() -> anyhow::Result<()> {
             vpc_subnet,
             gateway_mac,
             gateway_ip,
-            bsvc_addr,
-            bsvc_vni,
-            bsvc_mac,
             vpc_vni,
             src_underlay_addr,
             dhcp,
@@ -595,11 +598,6 @@ fn main() -> anyhow::Result<()> {
                 gateway_mac,
                 vni: vpc_vni,
                 phys_ip: src_underlay_addr,
-                boundary_services: BoundaryServices {
-                    ip: bsvc_addr,
-                    vni: bsvc_vni,
-                    mac: bsvc_mac,
-                },
             };
 
             hdl.create_xde(&name, cfg, dhcp.into(), passthrough)?;
@@ -624,6 +622,32 @@ fn main() -> anyhow::Result<()> {
             hdl.set_v2p(&req)?;
         }
 
+        Command::SetV2B { prefix, tunnel_endpoint } => {
+            let hdl = opteadm::OpteAdm::open(OpteAdm::XDE_CTL)?;
+            let tep = tunnel_endpoint
+                .into_iter()
+                .map(|ip| TunnelEndpoint {
+                    ip,
+                    vni: Vni::new(BOUNDARY_SERVICES_VNI).unwrap(),
+                })
+                .collect();
+            let req = SetVirt2BoundaryReq { vip: prefix, tep };
+            hdl.set_v2b(&req)?;
+        }
+
+        Command::ClearV2B { prefix, tunnel_endpoint } => {
+            let hdl = opteadm::OpteAdm::open(OpteAdm::XDE_CTL)?;
+            let tep = tunnel_endpoint
+                .into_iter()
+                .map(|ip| TunnelEndpoint {
+                    ip,
+                    vni: Vni::new(BOUNDARY_SERVICES_VNI).unwrap(),
+                })
+                .collect();
+            let req = ClearVirt2BoundaryReq { vip: prefix, tep };
+            hdl.clear_v2b(&req)?;
+        }
+
         Command::AddRouterEntry { port, dest, target } => {
             let req = AddRouterEntryReq { port_name: port, dest, target };
             hdl.add_router_entry(&req)?;

Cargo.toml

@@ -14,6 +14,7 @@ use opte_ioctl::run_cmd_ioctl;
 use opte_ioctl::Error;
 use oxide_vpc::api::AddFwRuleReq;
 use oxide_vpc::api::AddRouterEntryReq;
+use oxide_vpc::api::ClearVirt2BoundaryReq;
 use oxide_vpc::api::CreateXdeReq;
 use oxide_vpc::api::DeleteXdeReq;
 use oxide_vpc::api::DhcpCfg;
@@ -22,6 +23,7 @@ use oxide_vpc::api::ListPortsResp;
 use oxide_vpc::api::RemFwRuleReq;
 use oxide_vpc::api::SetExternalIpsReq;
 use oxide_vpc::api::SetFwRulesReq;
+use oxide_vpc::api::SetVirt2BoundaryReq;
 use oxide_vpc::api::SetVirt2PhysReq;
 use oxide_vpc::api::VpcCfg;
 use oxide_vpc::engine::overlay;
@@ -226,6 +228,29 @@ impl OpteAdm {
         )
     }
 
+    pub fn set_v2b(&self, req: &SetVirt2BoundaryReq) -> Result<NoResp, Error> {
+        let cmd = OpteCmd::SetVirt2Boundary;
+        run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
+    }
+
+    pub fn clear_v2b(
+        &self,
+        req: &ClearVirt2BoundaryReq,
+    ) -> Result<NoResp, Error> {
+        let cmd = OpteCmd::ClearVirt2Boundary;
+        run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
+    }
+
+    /// Dump the Virtual-to-Boundary mappings.
+    pub fn dump_v2b(&self) -> Result<overlay::DumpVirt2BoundaryResp, Error> {
+        let cmd = OpteCmd::DumpVirt2Boundary;
+        run_cmd_ioctl(
+            self.device.as_raw_fd(),
+            cmd,
+            Some(&overlay::DumpVirt2BoundaryReq { unused: 99 }),
+        )
+    }
+
     pub fn add_router_entry(
         &self,
         req: &AddRouterEntryReq,

bin/opteadm/src/bin/opteadm.rs

@@ -20,23 +20,26 @@ pub const XDE_IOC_OPTE_CMD: i32 = XDE_IOC as i32 | 0x01;
 #[derive(Clone, Copy, Debug)]
 #[repr(C)]
 pub enum OpteCmd {
-    ListPorts = 1,       // list all ports
-    AddFwRule = 20,      // add firewall rule
-    RemFwRule = 21,      // remove firewall rule
-    SetFwRules = 22,     // set/replace all firewall rules at once
-    DumpTcpFlows = 30,   // dump TCP flows
-    DumpLayer = 31,      // dump the specified Layer
-    DumpUft = 32,        // dump the Unified Flow Table
-    ListLayers = 33,     // list the layers on a given port
-    ClearUft = 40,       // clear the UFT
-    ClearLft = 41,       // clear the given Layer's Flow Table
-    SetVirt2Phys = 50,   // set a v2p mapping
-    DumpVirt2Phys = 51,  // dump the v2p mappings
-    AddRouterEntry = 60, // add a router entry for IP dest
-    CreateXde = 70,      // create a new xde device
-    DeleteXde = 71,      // delete an xde device
-    SetXdeUnderlay = 72, // set xde underlay devices
-    SetExternalIps = 80, // set xde external IPs for a port
+    ListPorts = 1,           // list all ports
+    AddFwRule = 20,          // add firewall rule
+    RemFwRule = 21,          // remove firewall rule
+    SetFwRules = 22,         // set/replace all firewall rules at once
+    DumpTcpFlows = 30,       // dump TCP flows
+    DumpLayer = 31,          // dump the specified Layer
+    DumpUft = 32,            // dump the Unified Flow Table
+    ListLayers = 33,         // list the layers on a given port
+    ClearUft = 40,           // clear the UFT
+    ClearLft = 41,           // clear the given Layer's Flow Table
+    SetVirt2Phys = 50,       // set a v2p mapping
+    DumpVirt2Phys = 51,      // dump the v2p mappings
+    SetVirt2Boundary = 52,   // set a v2b mapping
+    ClearVirt2Boundary = 53, // clear a v2b mapping
+    DumpVirt2Boundary = 54,  // dump the v2b mappings
+    AddRouterEntry = 60,     // add a router entry for IP dest
+    CreateXde = 70,          // create a new xde device
+    DeleteXde = 71,          // delete an xde device
+    SetXdeUnderlay = 72,     // set xde underlay devices
+    SetExternalIps = 80,     // set xde external IPs for a port
 }
 
 impl TryFrom<c_int> for OpteCmd {
@@ -56,6 +59,9 @@ impl TryFrom<c_int> for OpteCmd {
             41 => Ok(Self::ClearLft),
             50 => Ok(Self::SetVirt2Phys),
             51 => Ok(Self::DumpVirt2Phys),
+            52 => Ok(Self::SetVirt2Boundary),
+            53 => Ok(Self::ClearVirt2Boundary),
+            54 => Ok(Self::DumpVirt2Boundary),
             60 => Ok(Self::AddRouterEntry),
             70 => Ok(Self::CreateXde),
             71 => Ok(Self::DeleteXde),