Move underlay config from static conf to ioctl (#93)

Fixes #87

This PR moves getting underlay devices from static driver configuration in the driver attach routine to an ioctl that allows the driver to be initially loaded without an underlay config and then configured with underlay devices later.

In addition to the ioctl plumbing, the XdeState data structure is updated to include an Option<UnderlayState> to force users of XdeState to deal with the fact that the underlay may not be configured yet.

Author: rcgoodfellow

illumos-ddi-dki/src/lib.rs

@@ -533,6 +533,7 @@ pub const EAGAIN: c_int = 11;
 pub const ENOMEM: c_int = 12;
 pub const EFAULT: c_int = 14;
 pub const EBUSY: c_int = 16;
+pub const EEXIST: c_int = 17;
 pub const EINVAL: c_int = 22;
 pub const EPIPE: c_int = 32;
 pub const ENOMSG: c_int = 35;

opte-core/src/ioctl.rs

@@ -62,6 +62,7 @@ pub enum OpteCmd {
     AddRouterEntryIpv4 = 60, // add a router entry for IPv4 dest
     CreateXde = 70,          // create a new xde device
     DeleteXde = 71,          // delete an xde device
+    SetXdeUnderlay = 72,     // set xde underlay devices
 }
 
 impl TryFrom<c_int> for OpteCmd {
@@ -82,6 +83,7 @@ impl TryFrom<c_int> for OpteCmd {
             60 => Ok(Self::AddRouterEntryIpv4),
             70 => Ok(Self::CreateXde),
             71 => Ok(Self::DeleteXde),
+            72 => Ok(Self::SetXdeUnderlay),
             _ => Err(()),
         }
     }
@@ -357,6 +359,13 @@ pub struct DeleteXdeReq {
     pub xde_devname: String,
 }
 
+/// Set the underlay devices used by the xde kernel module
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct SetXdeUnderlayReq {
+    pub u1: String,
+    pub u2: String,
+}
+
 #[derive(Debug, Default, Deserialize, Serialize)]
 pub struct NoResp {
     pub unused: u64,

opteadm/src/lib.rs

@@ -13,6 +13,7 @@ use opte_core::ether::EtherAddr;
 use opte_core::geneve::Vni;
 use opte_core::ioctl::{
     self as api, CmdOk, CreateXdeReq, DeleteXdeReq, NoResp, OpteCmd,
+    SetXdeUnderlayReq,
 };
 use opte_core::ip4::Ipv4Addr;
 use opte_core::ip6::Ipv6Addr;
@@ -146,6 +147,17 @@ impl OpteAdm {
         Ok(resp)
     }
 
+    /// Set xde underlay devices
+    pub fn set_xde_underlay(
+        &self,
+        u1: &str,
+        u2: &str,
+    ) -> Result<NoResp, Error> {
+        let req = SetXdeUnderlayReq { u1: u1.into(), u2: u2.into() };
+        let cmd = OpteCmd::SetXdeUnderlay;
+        run_cmd_ioctl(self.device.as_raw_fd(), cmd, &req)
+    }
+
     /// Add a firewall rule
     pub fn add_firewall_rule(
         &self,

opteadm/src/main.rs

@@ -110,6 +110,9 @@ enum Command {
     /// Delete an xde device
     DeleteXde { name: String },
 
+    /// Set up xde underlay devices
+    SetXdeUnderlay { u1: String, u2: String },
+
     /// Set a virtual-to-physical mapping
     SetV2P {
         vpc_ip4: Ipv4Addr,
@@ -641,6 +644,11 @@ fn main() {
             let _ = hdl.delete_xde(&name).unwrap_or_die();
         }
 
+        Command::SetXdeUnderlay { u1, u2 } => {
+            let hdl = opteadm::OpteAdm::open(OpteAdm::DLD_CTL).unwrap_or_die();
+            let _ = hdl.set_xde_underlay(&u1, &u2).unwrap_or_die();
+        }
+
         Command::RmFwRule { port, direction, id } => {
             let hdl = opteadm::OpteAdm::open(OpteAdm::DLD_CTL).unwrap_or_die();
             let request = RemFwRuleReq { port_name: port, dir: direction, id };