No LFTs yet, no expiry.

Rather a lot of plumbing

Author: FelixMcFelix

lib/opte-test-utils/src/lib.rs

@@ -269,11 +269,11 @@ fn oxide_net_builder(
     let dhcp = base_dhcp_config();
 
     firewall::setup(&mut pb, fw_limit).expect("failed to add firewall layer");
-    gateway::setup(&pb, cfg, vpc_map, fw_limit, &dhcp)
+    gateway::setup(&mut pb, cfg, vpc_map, fw_limit, &dhcp)
         .expect("failed to setup gateway layer");
-    router::setup(&pb, cfg, one_limit).expect("failed to add router layer");
+    router::setup(&mut pb, cfg, one_limit).expect("failed to add router layer");
     nat::setup(&mut pb, cfg, snat_limit).expect("failed to add nat layer");
-    overlay::setup(&pb, cfg, v2p, v2b, one_limit)
+    overlay::setup(&mut pb, cfg, v2p, v2b, one_limit)
         .expect("failed to add overlay layer");
     pb
 }

lib/opte/src/engine/layer.rs

@@ -17,6 +17,7 @@ use super::packet::InnerFlowId;
 use super::packet::MblkFullParsed;
 use super::packet::MblkPacketData;
 use super::packet::Packet;
+use super::port::PortBuilder;
 use super::port::Transforms;
 use super::port::meta::ActionMeta;
 use super::rule;
@@ -28,6 +29,8 @@ use super::rule::GenBtError;
 use super::rule::HdrTransformError;
 use super::rule::Rule;
 use super::rule::ht_probe;
+use super::stat::StatTree;
+use super::stat::TableStat;
 use crate::ExecCtx;
 use crate::LogLevel;
 use crate::api::DumpLayerResp;
@@ -56,6 +59,7 @@ use opte_api::Direction;
 use opte_api::RuleDump;
 use opte_api::RuleId;
 use opte_api::RuleTableEntryDump;
+use uuid::Uuid;
 
 #[derive(Debug)]
 pub enum LayerError {
@@ -353,8 +357,9 @@ pub enum EntryState {
 /// reasonable to open this up to be any [`Action`], if such a use
 /// case were to present itself. For now, we stay conservative, and
 /// supply only what the current consumers need.
-#[derive(Copy, Clone, Debug)]
+#[derive(Copy, Clone, Debug, Default)]
 pub enum DefaultAction {
+    #[default]
     Allow,
     StatefulAllow,
     Deny,
@@ -407,7 +412,7 @@ impl Display for ActionDescEntry {
 ///
 /// This describes the actions a layer's rules can take as well as the
 /// [`DefaultAction`] to take when a rule doesn't match.
-#[derive(Debug)]
+#[derive(Debug, Default)]
 pub struct LayerActions {
     /// The list of actions shared among the layer's rules. An action
     /// doesn't have to be shared, each rule is free to create its
@@ -419,9 +424,15 @@ pub struct LayerActions {
     /// direction.
     pub default_in: DefaultAction,
 
+    /// The stats ID to attach to the default-in action.
+    pub default_in_stat_id: Option<Uuid>,
+
     /// The default action to take if no rule matches in the outbound
     /// direction.
     pub default_out: DefaultAction,
+
+    /// The stats ID to attach to the default-in action.
+    pub default_out_stat_id: Option<Uuid>,
 }
 
 #[derive(KStatProvider)]
@@ -503,8 +514,10 @@ pub struct Layer {
     actions: Vec<Action>,
     default_in: DefaultAction,
     default_in_hits: u64,
+    default_in_stat: Arc<TableStat>,
     default_out: DefaultAction,
     default_out_hits: u64,
+    default_out_stat: Arc<TableStat>,
     ft: LayerFlowTable,
     ft_cstr: CString,
     rules_in: RuleTable,
@@ -518,15 +531,20 @@ impl Layer {
         self.actions.get(idx).cloned()
     }
 
-    pub fn add_rule(&mut self, dir: Direction, rule: Rule<Finalized>) {
+    pub fn add_rule(
+        &mut self,
+        dir: Direction,
+        rule: Rule<Finalized>,
+        stats: &mut StatTree,
+    ) {
         match dir {
             Direction::Out => {
-                self.rules_out.add(rule);
+                self.rules_out.add(rule, stats);
                 self.stats.vals.out_rules += 1;
             }
 
             Direction::In => {
-                self.rules_in.add(rule);
+                self.rules_in.add(rule, stats);
                 self.stats.vals.in_rules += 1;
             }
         }
@@ -737,18 +755,24 @@ impl Layer {
 
     pub fn new(
         name: &'static str,
-        port: &str,
+        port: &mut PortBuilder,
         actions: LayerActions,
         ft_limit: NonZeroU32,
     ) -> Self {
-        let port_c = CString::new(port).unwrap();
+        let stats = port.stats_mut();
+        let default_in_stat = stats.root(actions.default_in_stat_id);
+        let default_out_stat = stats.root(actions.default_out_stat_id);
+
+        let port_name = port.name();
+
+        let port_c = CString::new(port_name).unwrap();
         let name_c = CString::new(name).unwrap();
 
         // Unwrap: We know this is fine because the stat names are
         // generated from the LayerStats structure.
         let stats = KStatNamed::new(
             "xde",
-            &format!("{}_{}", port, name),
+            &format!("{}_{}", port_name, name),
             LayerStats::new(),
         )
         .unwrap();
@@ -759,15 +783,17 @@ impl Layer {
             actions: actions.actions,
             default_in: actions.default_in,
             default_in_hits: 0,
+            default_in_stat,
             default_out: actions.default_out,
             default_out_hits: 0,
+            default_out_stat,
             name,
             name_c,
             port_c,
-            ft: LayerFlowTable::new(port, name, ft_limit),
+            ft: LayerFlowTable::new(port_name, name, ft_limit),
             ft_cstr: CString::new(format!("ft-{}", name)).unwrap(),
-            rules_in: RuleTable::new(port, name, Direction::In),
-            rules_out: RuleTable::new(port, name, Direction::Out),
+            rules_in: RuleTable::new(port_name, name, Direction::In),
+            rules_out: RuleTable::new(port_name, name, Direction::Out),
             rt_cstr: CString::new(format!("rt-{}", name)).unwrap(),
             stats,
         }
@@ -795,12 +821,12 @@ impl Layer {
         xforms: &mut Transforms,
         ameta: &mut ActionMeta,
     ) -> result::Result<LayerResult, LayerError> {
-        use Direction::*;
         let flow_before = *pkt.flow();
         self.layer_process_entry_probe(dir, pkt.flow());
+        pkt.meta_mut().stats.new_layer();
         let res = match dir {
-            Out => self.process_out(ectx, pkt, xforms, ameta),
-            In => self.process_in(ectx, pkt, xforms, ameta),
+            Direction::Out => self.process_out(ectx, pkt, xforms, ameta),
+            Direction::In => self.process_in(ectx, pkt, xforms, ameta),
         };
         self.layer_process_return_probe(dir, &flow_before, pkt.flow(), &res);
         res
@@ -888,15 +914,21 @@ impl Layer {
         self.stats.vals.in_lft_miss += 1;
         let rule = self.rules_in.find_match(pkt.flow(), pkt.meta(), ameta);
 
-        let action = if let Some(rule) = rule {
+        let (action, stat) = if let Some(rule) = rule {
             self.stats.vals.in_rule_match += 1;
-            rule.action()
+            (rule.rule.action(), rule.stat.clone())
         } else {
             self.stats.vals.in_rule_nomatch += 1;
             self.default_in_hits += 1;
-            self.default_in.into()
+            (self.default_in.into(), self.default_in_stat.clone())
         };
 
+        // No LFT to account for.
+        let mut stat = Some(stat);
+        if !matches!(action, Action::StatefulAllow | Action::Stateful(_)) {
+            pkt.meta_mut().stats.push(stat.take().unwrap());
+        }
+
         match action {
             Action::Allow => Ok(LayerResult::Allow),
 
@@ -909,6 +941,8 @@ impl Layer {
                     });
                 }
 
+                // TODO: how on earth are we plumbing StatTree into here??
+
                 // The outbound flow ID mirrors the inbound. Remember,
                 // the "top" of layer represents how the client sees
                 // the traffic, and the "bottom" of the layer
@@ -1019,6 +1053,8 @@ impl Layer {
                     });
                 }
 
+                // TODO: how on earth are we plumbing StatTree into here??
+
                 let desc = match action.gen_desc(pkt.flow(), pkt, ameta) {
                     Ok(aord) => match aord {
                         AllowOrDeny::Allow(desc) => desc,
@@ -1175,15 +1211,21 @@ impl Layer {
         self.stats.vals.out_lft_miss += 1;
         let rule = self.rules_out.find_match(pkt.flow(), pkt.meta(), ameta);
 
-        let action = if let Some(rule) = rule {
+        let (action, stat) = if let Some(rule) = rule {
             self.stats.vals.out_rule_match += 1;
-            rule.action()
+            (rule.rule.action(), rule.stat.clone())
         } else {
             self.stats.vals.out_rule_nomatch += 1;
             self.default_out_hits += 1;
-            self.default_out.into()
+            (self.default_out.into(), self.default_out_stat.clone())
         };
 
+        // No LFT to account for.
+        let mut stat = Some(stat);
+        if !matches!(action, Action::StatefulAllow | Action::Stateful(_)) {
+            pkt.meta_mut().stats.push(stat.take().unwrap());
+        }
+
         match action {
             Action::Allow => Ok(LayerResult::Allow),
 
@@ -1196,6 +1238,8 @@ impl Layer {
                     });
                 }
 
+                // TODO: how on earth are we plumbing StatTree into here??
+
                 // The inbound flow ID must be calculated _after_ the
                 // header transformation. Remember, the "top"
                 // (outbound) of layer represents how the client sees
@@ -1308,6 +1352,8 @@ impl Layer {
                     });
                 }
 
+                // TODO: how on earth are we plumbing StatTree into here??
+
                 let desc = match action.gen_desc(pkt.flow(), pkt, ameta) {
                     Ok(aord) => match aord {
                         AllowOrDeny::Allow(desc) => desc,
@@ -1492,9 +1538,10 @@ impl Layer {
         &mut self,
         in_rules: Vec<Rule<Finalized>>,
         out_rules: Vec<Rule<Finalized>>,
+        stats: &mut StatTree,
     ) {
         self.ft.clear();
-        self.set_rules_core(in_rules, out_rules);
+        self.set_rules_core(in_rules, out_rules, stats);
     }
 
     /// Set all rules at once without clearing the flow table.
@@ -1505,18 +1552,20 @@ impl Layer {
         &mut self,
         in_rules: Vec<Rule<Finalized>>,
         out_rules: Vec<Rule<Finalized>>,
+        stats: &mut StatTree,
     ) {
         self.ft.mark_dirty();
-        self.set_rules_core(in_rules, out_rules);
+        self.set_rules_core(in_rules, out_rules, stats);
     }
 
     fn set_rules_core(
         &mut self,
         in_rules: Vec<Rule<Finalized>>,
         out_rules: Vec<Rule<Finalized>>,
+        stats: &mut StatTree,
     ) {
-        self.rules_in.set_rules(in_rules);
-        self.rules_out.set_rules(out_rules);
+        self.rules_in.set_rules(in_rules, stats);
+        self.rules_out.set_rules(out_rules, stats);
         self.stats.vals.set_rules_called += 1;
         self.stats.vals.in_rules.set(self.rules_in.num_rules() as u64);
         self.stats.vals.out_rules.set(self.rules_out.num_rules() as u64);
@@ -1532,6 +1581,7 @@ struct RuleTableEntry {
     id: RuleId,
     hits: u64,
     rule: Rule<rule::Finalized>,
+    stat: Arc<TableStat>,
 }
 
 impl From<&RuleTableEntry> for RuleTableEntryDump {
@@ -1561,15 +1611,18 @@ pub enum RuleRemoveErr {
 }
 
 impl RuleTable {
-    fn add(&mut self, rule: Rule<rule::Finalized>) {
+    fn add(&mut self, rule: Rule<rule::Finalized>, stats: &mut StatTree) {
+        let stat = stats.root(rule.stat_id().copied());
         match self.find_pos(&rule) {
             RulePlace::End => {
-                let rte = RuleTableEntry { id: self.next_id, hits: 0, rule };
+                let rte =
+                    RuleTableEntry { id: self.next_id, hits: 0, rule, stat };
                 self.rules.push(rte);
             }
 
             RulePlace::Insert(idx) => {
-                let rte = RuleTableEntry { id: self.next_id, hits: 0, rule };
+                let rte =
+                    RuleTableEntry { id: self.next_id, hits: 0, rule, stat };
                 self.rules.insert(idx, rte);
             }
         }
@@ -1589,7 +1642,7 @@ impl RuleTable {
         ifid: &InnerFlowId,
         pmeta: &MblkPacketData,
         ameta: &ActionMeta,
-    ) -> Option<&Rule<rule::Finalized>> {
+    ) -> Option<&RuleTableEntry> {
         for rte in self.rules.iter_mut() {
             if rte.rule.is_match(pmeta, ameta) {
                 rte.hits += 1;
@@ -1600,7 +1653,7 @@ impl RuleTable {
                     ifid,
                     &rte.rule,
                 );
-                return Some(&rte.rule);
+                return Some(rte);
             }
         }
 
@@ -1738,10 +1791,14 @@ impl RuleTable {
         }
     }
 
-    pub fn set_rules(&mut self, new_rules: Vec<Rule<rule::Finalized>>) {
+    pub fn set_rules(
+        &mut self,
+        new_rules: Vec<Rule<rule::Finalized>>,
+        stats: &mut StatTree,
+    ) {
         self.rules.clear();
         for r in new_rules {
-            self.add(r);
+            self.add(r, stats);
         }
     }
 }
@@ -1825,6 +1882,7 @@ mod test {
         use crate::engine::predicate::Predicate;
         use crate::engine::rule;
 
+        let mut stats = StatTree::default();
         let mut rule_table = RuleTable::new("port", "test", Direction::Out);
         let mut rule = Rule::new(
             1,
@@ -1835,7 +1893,7 @@ mod test {
             Ipv4AddrMatch::Prefix(cidr),
         ]));
 
-        rule_table.add(rule.finalize());
+        rule_table.add(rule.finalize(), &mut stats);
 
         let mut test_pkt = MsgBlk::new_ethernet_pkt((
             Ethernet { ethertype: Ethertype::IPV4, ..Default::default() },