add ability to set all firewall rules atomically (#143)

Need UFT invalidation on layer/rule changes (#21)

Author: rzezeski

dtrace/opte-port-process.d

@@ -5,8 +5,8 @@
  */
 #include "common.h"
 
-#define	HDR_FMT		"%-12s %-3s %-43s %-18s %s\n"
-#define	LINE_FMT	"%-12s %-3s %-43s 0x%-16p %s\n"
+#define	HDR_FMT		"%-12s %-3s %-8s %-43s %-18s %s\n"
+#define	LINE_FMT	"%-12s %-3s %-8u %-43s 0x%-16p %s\n"
 
 BEGIN {
 	/*
@@ -18,19 +18,21 @@ BEGIN {
 	protos[17] = "UDP";
 	protos[255] = "XXX";
 
-	printf(HDR_FMT, "NAME", "DIR", "FLOW", "MBLK", "RESULT");
+	printf(HDR_FMT, "NAME", "DIR", "EPOCH", "FLOW", "MBLK", "RESULT");
 	num = 0;
 }
 
 port-process-return {
 	this->dir = stringof(arg0);
 	this->name = stringof(arg1);
 	this->flow = (flow_id_sdt_arg_t *)arg2;
-	this->mp = arg3;
-	this->res = stringof(arg4);
+	this->epoch = arg3;
+	this->mp = arg4;
+	this->res = stringof(arg5);
 
 	if (num >= 10) {
-		printf(HDR_FMT, "NAME", "DIR", "FLOW", "MBLK", "RESULT");
+		printf(HDR_FMT, "NAME", "DIR", "EPOCH", "FLOW", "MBLK",
+		    "RESULT");
 		num = 0;
 	}
 
@@ -43,13 +45,15 @@ port-process-return {
 
 port-process-return /this->af == AF_INET/ {
 	FLOW_FMT(this->s, this->flow);
-	printf(LINE_FMT, this->name, this->dir, this->s, this->mp, this->res);
+	printf(LINE_FMT, this->name, this->dir, this->epoch, this->s, this->mp,
+	    this->res);
 	num++;
 }
 
 port-process-return /this->af == AF_INET6/ {
 	FLOW_FMT6(this->s, this->flow);
-	printf(LINE_FMT,  this->name, this->dir, this->s, this->mp, this->res);
+	printf(LINE_FMT,  this->name, this->dir, this->epoch, this->s, this->mp,
+	    this->res);
 	num++;
 }
 

dtrace/opte-uft-invaildate.d

@@ -0,0 +1,57 @@
+/*
+ * Track UFT entry invalidations as they happen. An invalidation
+ * occurs when the port's epoch has move forward based on a rule
+ * change but the UFT entry is based on an older epoch; therefore it
+ * needs to be invalidated so that a new entry may be generated from
+ * the current rule set.
+ *
+ * dtrace -L ./lib -I . -Cqs ./opte-uft-invalidate.d
+ */
+#include "common.h"
+
+#define	HDR_FMT		"%-8s %-3s %-43s %s\n"
+#define	LINE_FMT	"%-8s %-3s %-43s %u\n"
+
+BEGIN {
+	/*
+	 * Use an associative array to stringify the protocol number.
+	 */
+	protos[1] = "ICMP";
+	protos[2] = "IGMP";
+	protos[6] = "TCP";
+	protos[17] = "UDP";
+	protos[255] = "XXX";
+
+	printf(HDR_FMT, "PORT", "DIR", "FLOW", "EPOCH");
+	num = 0;
+}
+
+flow-entry-invalidated {
+	this->dir = stringof(arg0);
+	this->port = stringof(arg1);
+	this->flow = (flow_id_sdt_arg_t *)arg2;
+	this->epoch = arg3;
+	this->af = this->flow->af;
+
+	if (num >= 10) {
+		printf(HDR_FMT, "PORT", "DIR", "FLOW", "EPOCH");
+		num = 0;
+	}
+
+	if (this->af != AF_INET && this->af != AF_INET6) {
+		printf("BAD ADDRESS FAMILY: %d\n", this->af);
+	}
+}
+
+ft-entry-invliadated /this->af == AF_INET/ {
+	FLOW_FMT(this->s, this->flow);
+	printf(LINE_FMT, this->dir, this->port, this->s, this->epoch);
+	num++;
+}
+
+ft-entry-invliadated /this->af == AF_INET6/ {
+	FLOW_FMT6(this->s, this->flow);
+	printf(LINE_FMT, this->dir, this->port, this->s, this->epoch);
+	num++;
+}
+

dtrace/usdt-port-process.d

@@ -1,23 +1,26 @@
-#define	HDR_FMT		"%-3s %-12s %-43s 0x%-16s %s\n"
-#define	LINE_FMT	"%-3s %-12s %-43s 0x%-16p %s\n"
+#define	HDR_FMT		"%-3s %-12s %-8s %-43s %-18s %s\n"
+#define	LINE_FMT	"%-3s %-12s %-8u %-43s 0x%-16p %s\n"
 
 BEGIN {
-	printf(HDR_FMT, "DIR", "NAME", "FLOW", "MBLK", "RESULT");
+	printf(HDR_FMT, "DIR", "NAME", "EPOCH", "FLOW", "MBLK", "RESULT");
 	num = 0;
 }
 
 port-process-return {
 	this->dir = json(copyinstr(arg0), "ok");
 	this->name = copyinstr(arg1);
 	this->flow = copyinstr(arg2);
-	this->mp = arg3;
-	this->res = copyinstr(arg4);
+	this->epoch = arg3;
+	this->mp = arg4;
+	this->res = copyinstr(arg5);
 	num++;
 
 	if (num >= 10) {
-		printf(HDR_FMT, "DIR", "NAME", "FLOW", "MBLK", "RESULT");
+		printf(HDR_FMT, "DIR", "NAME", "EPCOH", "FLOW", "MBLK",
+		    "RESULT");
 		num = 0;
 	}
 
-	printf(LINE_FMT, this->dir, this->name, this->flow, this->mp, this->res);
-}
+	printf(LINE_FMT, this->dir, this->name, this->epoch, this->flow,
+	    this->mp, this->res);
+}

dtrace/usdt-uft-invalidate.d

@@ -0,0 +1,22 @@
+#define	HDR_FMT		"%-8s %-3s %-43s %s\n"
+#define	LINE_FMT	"%-8s %-3s %-43s %u\n"
+
+BEGIN {
+	printf(HDR_FMT, "PORT", "DIR", "FLOW", "EPOCH");
+	num = 0;
+}
+
+uft-invalidate {
+	this->dir = json(copyinstr(arg0), "ok");
+	this->port = copyinstr(arg1);
+	this->flow = copyinstr(arg2);
+	this->epoch = arg3;
+	num++;
+
+	if (num >= 10) {
+		printf(HDR_FMT, "PORT", "DIR", "FLOW", "EPOCH");
+		num = 0;
+	}
+
+	printf(LINE_FMT, this->port, this->dir, this->flow, this->epoch);
+}

opte-ioctl/src/lib.rs

@@ -10,7 +10,7 @@ use opte::api::{
 };
 use opte::oxide_vpc::api::{
     AddRouterEntryIpv4Req, CreateXdeReq, DeleteXdeReq, ListPortsReq,
-    ListPortsResp, SetVirt2PhysReq,
+    ListPortsResp, SetFwRulesReq, SetVirt2PhysReq,
 };
 use serde::de::DeserializeOwned;
 use serde::Serialize;
@@ -165,6 +165,11 @@ impl OpteHdl {
         let cmd = OpteCmd::AddRouterEntryIpv4;
         run_cmd_ioctl(self.device.as_raw_fd(), cmd, &req)
     }
+
+    pub fn set_fw_rules(&self, req: &SetFwRulesReq) -> Result<NoResp, Error> {
+        let cmd = OpteCmd::SetFwRules;
+        run_cmd_ioctl(self.device.as_raw_fd(), cmd, &req)
+    }
 }
 
 #[cfg(target_os = "illumos")]

opte/Cargo.lock

@@ -30,7 +30,7 @@ postcard = { version = "0.7.0", features = ["alloc"], default-features = false }
 #
 # cargo test --features=usdt
 #
-usdt = { version = "*", optional = true }
+usdt = { version = "0.3.2", optional = true }
 zerocopy = "0.6.1"
 
 #
@@ -62,4 +62,5 @@ default-features = false
 features = ["alloc", "medium-ethernet", "proto-ipv4", "proto-ipv6", "proto-dhcpv4", "socket", "socket-raw"]
 
 [dev-dependencies]
+ctor = "0.1.22"
 pcap-parser = { version = "0.11.1", features = ["serialize"] }

opte/Cargo.toml

@@ -25,6 +25,7 @@ pub enum OpteCmd {
     ListPorts = 1,           // list all ports
     AddFwRule = 20,          // add firewall rule
     RemFwRule = 21,          // remove firewall rule
+    SetFwRules = 22,         // set/replace all firewall rules at once
     DumpTcpFlows = 30,       // dump TCP flows
     DumpLayer = 31,          // dump the specified Layer
     DumpUft = 32,            // dump the Unified Flow Table
@@ -46,6 +47,7 @@ impl TryFrom<c_int> for OpteCmd {
             1 => Ok(Self::ListPorts),
             20 => Ok(Self::AddFwRule),
             21 => Ok(Self::RemFwRule),
+            22 => Ok(Self::SetFwRules),
             30 => Ok(Self::DumpTcpFlows),
             31 => Ok(Self::DumpLayer),
             32 => Ok(Self::DumpUft),

opte/src/api/cmd.rs

@@ -91,7 +91,7 @@ where
         let entry = FlowEntry::new(state);
         match self.map.insert(flow_id.clone(), entry) {
             None => Ok(()),
-            Some(_) => return Err(OpteError::FlowExists(flow_id.to_string())),
+            Some(_) => Err(OpteError::FlowExists(flow_id.to_string())),
         }
     }
 
@@ -158,6 +158,10 @@ where
         self.map.len() as u32
     }
 
+    pub fn remove(&mut self, flow: &InnerFlowId) -> Option<FlowEntry<S>> {
+        self.map.remove(flow)
+    }
+
     pub fn ttl(&self) -> Ttl {
         self.ttl
     }
@@ -176,8 +180,6 @@ fn flow_expired_probe(port: &CString, name: &CString, flowid: &InnerFlowId) {
                 );
             }
         } else if #[cfg(feature = "usdt")] {
-            use std::arch::asm;
-
             let port_s = port.to_str().unwrap();
             let name_s = name.to_str().unwrap();
             crate::opte_provider::flow__expired!(
@@ -256,6 +258,14 @@ extern "C" {
         layer: uintptr_t,
         flowid: uintptr_t,
     );
+
+    pub fn __dtrace_probe_ft__entry__invalidated(
+        dir: uintptr_t,
+        port: uintptr_t,
+        layer: uintptr_t,
+        ifid: uintptr_t,
+        epoch: uintptr_t,
+    );
 }
 
 impl StateSummary for () {