Implement DefinitionChecksum infrastructure and MaterializedView rebuild system

Establishes comprehensive type schema change detection system enabling selective
MaterializedView rebuilds when only type definitions change, avoiding expensive
full data rebuilds.

**Core Infrastructure**:
- Add DefinitionChecksum trait with static compilation-time checksum resolution
- Create comprehensive DefinitionChecksum implementations for all MV-used types
- Build DefinitionChecksum derive macro supporting recursive type handling
- Add RebuildChangedDefinitionsRequest API type with versioned structure

**MaterializedView Integration**:
- Extend MaterializedView derive macro to generate definition checksums
- Update MV infrastructure to detect and rebuild only definition-changed views
- Implement compressed request handling for selective rebuild operations

**Code Organization**:
- Refactor frigg into deployment, kv_history, and workspace modules
- Split si_frontend_mv_types::index into change_set and deployment submodules

**Compression Algorithm Enhancements**:
- Support RebuildChangedDefinitions requests in compression pipeline
- Maintain contiguous update sequences while filtering definition rebuilds
- Prioritize full rebuilds when mixed with selective definition rebuilds

The system now distinguishes between data changes requiring full rebuilds and
schema changes needing only definition updates, significantly improving rebuild
efficiency for type system modifications.

The change set/workspace level MvIndex will be considered "missing"
until it has been upgraded to the V2 format, which will trigger full
rebuilds for the change sets in question.

The deployment level MvIndex will continue to ignore the checksum
information for now, until we have had time to make sure it has been
upgraded to the V2 format.

Co-authored-by: Jacob Helwig <jacob@technosorcery.net>
Signed-off-by: Nick Gerace <nick@systeminit.com>

Author: jhelwig

lib/edda-core/src/api_types.rs

@@ -1,4 +1,5 @@
 pub mod new_change_set_request;
+pub mod rebuild_changed_definitions_request;
 pub mod rebuild_request;
 pub mod update_request;
 

lib/edda-core/src/api_types/rebuild_changed_definitions_request.rs

@@ -0,0 +1,153 @@
+use acceptable::{
+    AllVersions,
+    Container,
+    CurrentContainer,
+    IntoContainer,
+    UpgradeError,
+};
+use serde::Deserialize;
+
+mod v1;
+
+pub use self::v1::RebuildChangedDefinitionsRequestV1;
+
+#[remain::sorted]
+#[derive(AllVersions, CurrentContainer, Clone, Debug, Deserialize, Eq, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub enum RebuildChangedDefinitionsRequestAllVersions {
+    #[acceptable(current)]
+    V1(RebuildChangedDefinitionsRequestV1),
+}
+
+impl IntoContainer for RebuildChangedDefinitionsRequestAllVersions {
+    type Container = RebuildChangedDefinitionsRequest;
+
+    fn into_container(self) -> Result<Self::Container, UpgradeError> {
+        match self {
+            Self::V1(inner) => Ok(Self::Container::new(inner)),
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::{
+        error::Error,
+        fmt,
+        fs::File,
+        io::{
+            self,
+            BufRead as _,
+            BufReader,
+            Read as _,
+        },
+        path::Path,
+    };
+
+    use serde::{
+        Serialize,
+        de::DeserializeOwned,
+    };
+
+    /// Tests that a versioned object will always serialize to the same representation, no matter
+    /// what future versions or changes to the object.
+    ///
+    /// NOTE: It is imperative that incremental refactorings do not lead to a version-incompatible
+    /// or serialize-incompatible change. If a test fails and it's because there is a diff to the
+    /// commiteed `.snap` file, this should be considered a failed refactoring. The remediation is
+    /// to *not* update the `.snap` file but rather to fix the code so that the `.snap` format is
+    /// 100% preserved.
+    pub fn assert_serialize(name: &str, version: u64, serialize: impl Serialize) {
+        insta::with_settings!({
+            snapshot_path => format!("rebuild_changed_definitions_request/snapshots-v{version}"),
+            prepend_module_to_snapshot => false,
+            omit_expression => true,
+            description => concat!(
+                "\n",
+                "\n",
+                "!!!\n",
+                "!!! System Initiative Developers:\n",
+                "!!!\n",
+                "!!! IMPORTANT:\n",
+                "!!!\n",
+                "!!!     The contents of this snapshot should *never* be modified as it\n",
+                "!!!     represents the serialization of a versioned Rust type. If a tests fails\n",
+                "!!!     with this warning, then something about a Rust type has changed\n",
+                "!!!     the wire serialization of this type and would represent a potential\n",
+                "!!!     production outage or data corruption.\n",
+                "!!!\n",
+                "!!!     Consider this an erroneous behavioral change of the Rust code and *not*\n",
+                "!!!     an out-of-date snapshot or fixture!\n",
+                "!!!\n",
+                "\n",
+                "\n",
+            ),
+        }, {
+            insta::assert_json_snapshot!(name, serialize);
+        });
+    }
+
+    pub fn assert_deserialize<T>(snapshot_name: &str, version: u64, expected: T)
+    where
+        T: fmt::Debug + DeserializeOwned + PartialEq,
+    {
+        let actual: T = read_from_snapshot(
+            snapshot_name,
+            &format!("rebuild_changed_definitions_request/snapshots-v{version}"),
+        )
+        .expect("failed to deserialize from snapshot");
+
+        assert_eq!(actual, expected);
+    }
+
+    fn read_from_snapshot<T>(name: &str, path: &str) -> Result<T, Box<dyn Error>>
+    where
+        T: fmt::Debug + DeserializeOwned + PartialEq,
+    {
+        let glob = format!("{path}/{name}.snap");
+
+        let mut maybe_obj_result = None;
+        insta::glob!(&glob, |path| {
+            let bytes = read_snapshot_content(path).unwrap();
+
+            maybe_obj_result = Some(serde_json::from_slice(&bytes).map_err(Into::into));
+        });
+
+        match maybe_obj_result {
+            Some(object_result) => object_result,
+            None => Err(Box::new(io::Error::other(format!(
+                "snapshot not found: {glob}"
+            )))),
+        }
+    }
+
+    // Implementation is adapted from the [`insta::Snapshot::from_file`] function.
+    //
+    // See: <https://github.com/mitsuhiko/insta/blob/62bb0a3/insta/src/snapshot.rs#L339-L421>
+    fn read_snapshot_content(path: &Path) -> Result<Vec<u8>, Box<dyn Error>> {
+        let mut f = BufReader::new(File::open(path)?);
+
+        // Skip through the snapshot metadata, that being the first YAML document, where YAML
+        // documents are delimited with a `"---"` line
+        {
+            let mut buf = String::new();
+            f.read_line(&mut buf)?;
+            if buf.trim_end() == "---" {
+                loop {
+                    let read = f.read_line(&mut buf)?;
+                    if read == 0 {
+                        break;
+                    }
+                    if buf[buf.len() - read..].trim_end() == "---" {
+                        break;
+                    }
+                }
+            }
+        }
+
+        let mut buf = Vec::new();
+        f.read_to_end(&mut buf)?;
+
+        Ok(buf)
+    }
+}

lib/edda-core/src/api_types/rebuild_changed_definitions_request/snapshots-v1/serialized.snap

@@ -0,0 +1,7 @@
+---
+source: lib/edda-core/src/api_types/rebuild_changed_definitions_request.rs
+description: "\n\n!!!\n!!! System Initiative Developers:\n!!!\n!!! IMPORTANT:\n!!!\n!!!     The contents of this snapshot should *never* be modified as it\n!!!     represents the serialization of a versioned Rust type. If a tests fails\n!!!     with this warning, then something about a Rust type has changed\n!!!     the wire serialization of this type and would represent a potential\n!!!     production outage or data corruption.\n!!!\n!!!     Consider this an erroneous behavioral change of the Rust code and *not*\n!!!     an out-of-date snapshot or fixture!\n!!!\n\n\n"
+---
+{
+  "id": "01JQCVVDHXYX6S9YCV773R13MG"
+}

lib/edda-core/src/api_types/rebuild_changed_definitions_request/v1.rs

@@ -0,0 +1,43 @@
+use acceptable::{
+    RequestId,
+    Versioned,
+};
+use serde::{
+    Deserialize,
+    Serialize,
+};
+
+#[derive(Clone, Debug, Deserialize, Eq, Serialize, PartialEq, Versioned)]
+#[serde(rename_all = "camelCase")]
+#[acceptable(version = 1)]
+// NOTE: **do not modify this datatype--it represents a historically stable, versioned request**
+pub struct RebuildChangedDefinitionsRequestV1 {
+    pub id: RequestId,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        super::test::*,
+        *,
+    };
+
+    const SNAPSHOT_NAME: &str = "serialized";
+    const VERSION: u64 = 1;
+
+    fn msg() -> RebuildChangedDefinitionsRequestV1 {
+        RebuildChangedDefinitionsRequestV1 {
+            id: "01JQCVVDHXYX6S9YCV773R13MG".parse().unwrap(),
+        }
+    }
+
+    #[test]
+    fn serialize() {
+        assert_serialize(SNAPSHOT_NAME, VERSION, msg());
+    }
+
+    #[test]
+    fn deserialize() {
+        assert_deserialize(SNAPSHOT_NAME, VERSION, msg());
+    }
+}