Merge pull request #7187 from systeminit/jhelwig/eng-3201-implement-intelligent-materializedview-rebuild-system

Implement intelligent materializedview rebuild system on schema changes

Author: nickgerace

lib/edda-core/src/api_types.rs

@@ -1,4 +1,5 @@
 pub mod new_change_set_request;
+pub mod rebuild_changed_definitions_request;
 pub mod rebuild_request;
 pub mod update_request;
 

lib/edda-core/src/api_types/rebuild_changed_definitions_request.rs

@@ -0,0 +1,153 @@
+use acceptable::{
+    AllVersions,
+    Container,
+    CurrentContainer,
+    IntoContainer,
+    UpgradeError,
+};
+use serde::Deserialize;
+
+mod v1;
+
+pub use self::v1::RebuildChangedDefinitionsRequestV1;
+
+#[remain::sorted]
+#[derive(AllVersions, CurrentContainer, Clone, Debug, Deserialize, Eq, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub enum RebuildChangedDefinitionsRequestAllVersions {
+    #[acceptable(current)]
+    V1(RebuildChangedDefinitionsRequestV1),
+}
+
+impl IntoContainer for RebuildChangedDefinitionsRequestAllVersions {
+    type Container = RebuildChangedDefinitionsRequest;
+
+    fn into_container(self) -> Result<Self::Container, UpgradeError> {
+        match self {
+            Self::V1(inner) => Ok(Self::Container::new(inner)),
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::{
+        error::Error,
+        fmt,
+        fs::File,
+        io::{
+            self,
+            BufRead as _,
+            BufReader,
+            Read as _,
+        },
+        path::Path,
+    };
+
+    use serde::{
+        Serialize,
+        de::DeserializeOwned,
+    };
+
+    /// Tests that a versioned object will always serialize to the same representation, no matter
+    /// what future versions or changes to the object.
+    ///
+    /// NOTE: It is imperative that incremental refactorings do not lead to a version-incompatible
+    /// or serialize-incompatible change. If a test fails and it's because there is a diff to the
+    /// commiteed `.snap` file, this should be considered a failed refactoring. The remediation is
+    /// to *not* update the `.snap` file but rather to fix the code so that the `.snap` format is
+    /// 100% preserved.
+    pub fn assert_serialize(name: &str, version: u64, serialize: impl Serialize) {
+        insta::with_settings!({
+            snapshot_path => format!("rebuild_changed_definitions_request/snapshots-v{version}"),
+            prepend_module_to_snapshot => false,
+            omit_expression => true,
+            description => concat!(
+                "\n",
+                "\n",
+                "!!!\n",
+                "!!! System Initiative Developers:\n",
+                "!!!\n",
+                "!!! IMPORTANT:\n",
+                "!!!\n",
+                "!!!     The contents of this snapshot should *never* be modified as it\n",
+                "!!!     represents the serialization of a versioned Rust type. If a tests fails\n",
+                "!!!     with this warning, then something about a Rust type has changed\n",
+                "!!!     the wire serialization of this type and would represent a potential\n",
+                "!!!     production outage or data corruption.\n",
+                "!!!\n",
+                "!!!     Consider this an erroneous behavioral change of the Rust code and *not*\n",
+                "!!!     an out-of-date snapshot or fixture!\n",
+                "!!!\n",
+                "\n",
+                "\n",
+            ),
+        }, {
+            insta::assert_json_snapshot!(name, serialize);
+        });
+    }
+
+    pub fn assert_deserialize<T>(snapshot_name: &str, version: u64, expected: T)
+    where
+        T: fmt::Debug + DeserializeOwned + PartialEq,
+    {
+        let actual: T = read_from_snapshot(
+            snapshot_name,
+            &format!("rebuild_changed_definitions_request/snapshots-v{version}"),
+        )
+        .expect("failed to deserialize from snapshot");
+
+        assert_eq!(actual, expected);
+    }
+
+    fn read_from_snapshot<T>(name: &str, path: &str) -> Result<T, Box<dyn Error>>
+    where
+        T: fmt::Debug + DeserializeOwned + PartialEq,
+    {
+        let glob = format!("{path}/{name}.snap");
+
+        let mut maybe_obj_result = None;
+        insta::glob!(&glob, |path| {
+            let bytes = read_snapshot_content(path).unwrap();
+
+            maybe_obj_result = Some(serde_json::from_slice(&bytes).map_err(Into::into));
+        });
+
+        match maybe_obj_result {
+            Some(object_result) => object_result,
+            None => Err(Box::new(io::Error::other(format!(
+                "snapshot not found: {glob}"
+            )))),
+        }
+    }
+
+    // Implementation is adapted from the [`insta::Snapshot::from_file`] function.
+    //
+    // See: <https://github.com/mitsuhiko/insta/blob/62bb0a3/insta/src/snapshot.rs#L339-L421>
+    fn read_snapshot_content(path: &Path) -> Result<Vec<u8>, Box<dyn Error>> {
+        let mut f = BufReader::new(File::open(path)?);
+
+        // Skip through the snapshot metadata, that being the first YAML document, where YAML
+        // documents are delimited with a `"---"` line
+        {
+            let mut buf = String::new();
+            f.read_line(&mut buf)?;
+            if buf.trim_end() == "---" {
+                loop {
+                    let read = f.read_line(&mut buf)?;
+                    if read == 0 {
+                        break;
+                    }
+                    if buf[buf.len() - read..].trim_end() == "---" {
+                        break;
+                    }
+                }
+            }
+        }
+
+        let mut buf = Vec::new();
+        f.read_to_end(&mut buf)?;
+
+        Ok(buf)
+    }
+}

lib/edda-core/src/api_types/rebuild_changed_definitions_request/snapshots-v1/serialized.snap

@@ -0,0 +1,7 @@
+---
+source: lib/edda-core/src/api_types/rebuild_changed_definitions_request.rs
+description: "\n\n!!!\n!!! System Initiative Developers:\n!!!\n!!! IMPORTANT:\n!!!\n!!!     The contents of this snapshot should *never* be modified as it\n!!!     represents the serialization of a versioned Rust type. If a tests fails\n!!!     with this warning, then something about a Rust type has changed\n!!!     the wire serialization of this type and would represent a potential\n!!!     production outage or data corruption.\n!!!\n!!!     Consider this an erroneous behavioral change of the Rust code and *not*\n!!!     an out-of-date snapshot or fixture!\n!!!\n\n\n"
+---
+{
+  "id": "01JQCVVDHXYX6S9YCV773R13MG"
+}

lib/edda-core/src/api_types/rebuild_changed_definitions_request/v1.rs

@@ -0,0 +1,43 @@
+use acceptable::{
+    RequestId,
+    Versioned,
+};
+use serde::{
+    Deserialize,
+    Serialize,
+};
+
+#[derive(Clone, Debug, Deserialize, Eq, Serialize, PartialEq, Versioned)]
+#[serde(rename_all = "camelCase")]
+#[acceptable(version = 1)]
+// NOTE: **do not modify this datatype--it represents a historically stable, versioned request**
+pub struct RebuildChangedDefinitionsRequestV1 {
+    pub id: RequestId,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        super::test::*,
+        *,
+    };
+
+    const SNAPSHOT_NAME: &str = "serialized";
+    const VERSION: u64 = 1;
+
+    fn msg() -> RebuildChangedDefinitionsRequestV1 {
+        RebuildChangedDefinitionsRequestV1 {
+            id: "01JQCVVDHXYX6S9YCV773R13MG".parse().unwrap(),
+        }
+    }
+
+    #[test]
+    fn serialize() {
+        assert_serialize(SNAPSHOT_NAME, VERSION, msg());
+    }
+
+    #[test]
+    fn deserialize() {
+        assert_deserialize(SNAPSHOT_NAME, VERSION, msg());
+    }
+}