New rnote file format, better compression and serialization, atomic file saving

Cargo.toml

@@ -25,9 +25,11 @@ anyhow = "1.0"
 approx = "0.5.1"
 async-fs = "2.1"
 base64 = "0.22.1"
+bitcode = { version = "0.6", features = ["serde"] }
 cairo-rs = { version = "0.20.1", features = ["v1_18", "png", "svg", "pdf"] }
 chrono = "0.4.38"
 clap = { version = "4.5", features = ["derive"] }
+crc32fast = { version = "1.4" }
 dialoguer = "0.11.0"
 flate2 = "1.0"
 fs_extra = "1.3"
@@ -83,7 +85,8 @@ url = "2.5"
 usvg = "0.44.0"
 winresource = "0.1.17"
 xmlwriter = "0.1.0"
-# Enabling feature > v20_9 causes linker errors on mingw
+zstd = { version = "0.13", features = ["zstdmt"] }
+
 # using a custom patched version to work on windows
 # as long as https://gitlab.gnome.org/World/Rust/poppler-rs/-/issues/1
 # is not fixed

crates/rnote-cli/src/cli.rs

@@ -1,14 +1,17 @@
 // Imports
-use crate::{export, import, test, thumbnail};
+use crate::{export, import, mutate, test, thumbnail};
+
 use anyhow::Context;
 use clap::Parser;
+use clap::builder::PossibleValuesParser;
 use rnote_compose::SplitOrder;
 use rnote_engine::SelectionCollision;
 use rnote_engine::engine::export::{
     DocExportFormat, DocPagesExportFormat, DocPagesExportPrefs, SelectionExportFormat,
     SelectionExportPrefs,
 };
 use rnote_engine::engine::import::XoppImportPrefs;
+use rnote_engine::fileformats::rnoteformat;
 use smol::fs::File;
 use smol::io::{AsyncReadExt, AsyncWriteExt};
 use std::path::{Path, PathBuf};
@@ -69,6 +72,27 @@ pub(crate) enum Command {
         #[arg(long, action = clap::ArgAction::SetTrue, global = true)]
         open: bool,
     },
+    /// Mutates one or more of the following for the specified Rnote file(s):{n}
+    /// compression method, compression level, serialization method, method lock
+    Mutate {
+        /// The rnote save file(s) to mutate
+        rnote_files: Vec<PathBuf>,
+        /// Keep the original rnote save file(s)
+        #[arg(long = "not-in-place", alias = "nip", action = clap::ArgAction::SetTrue)]
+        not_in_place: bool,
+        /// Sets method_lock to true, allowing a rnote save file to keep using non-default methods to serialize and compress itself
+        #[arg(short = 'l', long, action = clap::ArgAction::SetTrue, conflicts_with = "unlock")]
+        lock: bool,
+        /// Sets method_lock to false, coercing the file to use default methods on the next save
+        #[arg(short = 'u', long, action = clap::ArgAction::SetTrue, conflicts_with = "lock")]
+        unlock: bool,
+        #[arg(short = 's', long, action = clap::ArgAction::Set, value_parser = PossibleValuesParser::new(rnoteformat::SerializationMethod::VALID_STR_ARRAY))]
+        serialization_method: Option<String>,
+        #[arg(short = 'c', long, action = clap::ArgAction::Set, value_parser = PossibleValuesParser::new(rnoteformat::CompressionMethod::VALID_STR_ARRAY))]
+        compression_method: Option<String>,
+        #[arg(short = 'v', long, action = clap::ArgAction::Set)]
+        compression_level: Option<u8>,
+    },
     /// Generate rnote thumbail from a given file
     Thumbnail {
         /// Input rnote file
@@ -252,6 +276,28 @@ pub(crate) async fn run() -> anyhow::Result<()> {
             .await?;
             println!("Export finished!");
         }
+        Command::Mutate {
+            rnote_files,
+            not_in_place,
+            lock,
+            unlock,
+            serialization_method,
+            compression_method,
+            compression_level,
+        } => {
+            println!("Mutating..\n");
+            mutate::run_mutate(
+                rnote_files,
+                not_in_place,
+                lock,
+                unlock,
+                serialization_method,
+                compression_method,
+                compression_level,
+            )
+            .await?;
+            println!("Mutate finished!");
+        }
         Command::Thumbnail {
             rnote_file,
             size,

crates/rnote-cli/src/main.rs

@@ -6,6 +6,7 @@
 pub(crate) mod cli;
 pub(crate) mod export;
 pub(crate) mod import;
+pub(crate) mod mutate;
 pub(crate) mod test;
 pub(crate) mod thumbnail;
 pub(crate) mod validators;

crates/rnote-cli/src/meson.build

@@ -4,6 +4,7 @@ rnote_cli_sources = files(
     'export.rs',
     'import.rs',
     'main.rs',
+    'mutate.rs',
     'test.rs',
     'validators.rs',
 )

crates/rnote-cli/src/mutate.rs

@@ -0,0 +1,94 @@
+use rnote_engine::engine::EngineSnapshot;
+use rnote_engine::fileformats::FileFormatSaver;
+use rnote_engine::fileformats::rnoteformat::RnoteHeader;
+use rnote_engine::fileformats::{FileFormatLoader, rnoteformat::RnoteFile};
+use smol::{fs::OpenOptions, io::AsyncReadExt};
+use std::path::PathBuf;
+use std::str::FromStr;
+
+pub(crate) async fn run_mutate(
+    rnote_files: Vec<PathBuf>,
+    not_in_place: bool,
+    lock: bool,
+    unlock: bool,
+    serialization_method: Option<String>,
+    compression_method: Option<String>,
+    compression_level: Option<u8>,
+) -> anyhow::Result<()> {
+    let total_len = rnote_files.len();
+    let mut total_delta: f64 = 0.0;
+    for (idx, mut filepath) in rnote_files.into_iter().enumerate() {
+        println!("Working on file {} out of {}", idx + 1, total_len);
+
+        let file_read_operation = async {
+            let mut read_file = OpenOptions::new().read(true).open(&filepath).await?;
+
+            let mut bytes: Vec<u8> = {
+                match read_file.metadata().await {
+                    Ok(metadata) => {
+                        Vec::with_capacity(usize::try_from(metadata.len()).unwrap_or(usize::MAX))
+                    }
+                    Err(err) => {
+                        eprintln!("Failed to read file metadata, '{err}'");
+                        Vec::new()
+                    }
+                }
+            };
+
+            read_file.read_to_end(&mut bytes).await?;
+            Ok::<Vec<u8>, anyhow::Error>(bytes)
+        };
+        let bytes = file_read_operation.await?;
+        let old_size_mb = bytes.len() as f64 / 1e6;
+        let rnote_file = RnoteFile::load_from_bytes(&bytes)?;
+
+        let serialization = if let Some(ref str) = serialization_method {
+            rnote_engine::fileformats::rnoteformat::SerializationMethod::from_str(str).unwrap()
+        } else {
+            rnote_file.header.serialization
+        };
+
+        let mut compression = if let Some(ref str) = compression_method {
+            rnote_engine::fileformats::rnoteformat::CompressionMethod::from_str(str).unwrap()
+        } else {
+            rnote_file.header.compression
+        };
+
+        if let Some(lvl) = compression_level {
+            compression.update_compression_integer(lvl)?;
+        }
+
+        let method_lock = (rnote_file.header.method_lock | lock) && !unlock;
+        let uc_data = serialization.serialize(&EngineSnapshot::try_from(rnote_file)?)?;
+        let uc_size = uc_data.len() as u64;
+        let data = compression.compress(uc_data)?;
+
+        let rnote_file = RnoteFile {
+            header: RnoteHeader {
+                serialization,
+                compression,
+                uc_size,
+                method_lock,
+            },
+            body: data,
+        };
+
+        if not_in_place {
+            let file_stem = filepath
+                .file_stem()
+                .ok_or_else(|| anyhow::anyhow!("File does not contain a valid file stem"))?
+                .to_str()
+                .ok_or_else(|| anyhow::anyhow!("File does not contain a valid file stem"))?;
+            filepath.set_file_name(format!("{}_mut.rnote", file_stem));
+        }
+
+        let data = rnote_file.save_as_bytes("")?;
+        let new_size_mb = data.len() as f64 / 1e6;
+        rnote_engine::utils::atomic_save_to_file(&filepath, &data).await?;
+
+        println!("{:.2} MB → {:.2} MB", old_size_mb, new_size_mb,);
+        total_delta += new_size_mb - old_size_mb;
+    }
+    println!("\n⇒ ∆ = {:.2} MB", total_delta);
+    Ok(())
+}

crates/rnote-engine/Cargo.toml

@@ -13,10 +13,13 @@ rnote-compose = { workspace = true }
 
 anyhow = { workspace = true }
 approx = { workspace = true }
+async-fs = { workspace = true }
 base64 = { workspace = true }
+bitcode = { workspace = true, features = ["serde"] }
 cairo-rs = { workspace = true }
 chrono = { workspace = true }
 clap = { workspace = true, optional = true }
+crc32fast = { workspace = true }
 flate2 = { workspace = true }
 futures = { workspace = true }
 geo = { workspace = true }
@@ -55,6 +58,7 @@ tracing = { workspace = true }
 unicode-segmentation = { workspace = true }
 usvg = { workspace = true }
 xmlwriter = { workspace = true }
+zstd = { workspace = true, features = ["zstdmt"] }
 # the long-term plan is to remove the gtk4 dependency entirely after switching to another renderer.
 gtk4 = { workspace = true, optional = true }
 

crates/rnote-engine/src/engine/export.rs

@@ -331,9 +331,7 @@ impl Engine {
         let engine_snapshot = self.take_snapshot();
         rayon::spawn(move || {
             let result = || -> anyhow::Result<Vec<u8>> {
-                let rnote_file = RnoteFile {
-                    engine_snapshot: ijson::to_value(&engine_snapshot)?,
-                };
+                let rnote_file = RnoteFile::try_from(&engine_snapshot)?;
                 rnote_file.save_as_bytes(&file_name)
             };
             if oneshot_sender.send(result()).is_err() {
@@ -353,6 +351,7 @@ impl Engine {
             penholder: self.penholder.clone_config(),
             import_prefs: self.import_prefs.clone_config(),
             export_prefs: self.export_prefs.clone_config(),
+            save_prefs: self.save_prefs.clone_from_engine_to_engineconfig(),
             pen_sounds: self.pen_sounds(),
             optimize_epd: self.optimize_epd(),
         }

crates/rnote-engine/src/engine/import.rs

@@ -162,6 +162,7 @@ impl Engine {
         self.penholder = engine_config.penholder;
         self.import_prefs = engine_config.import_prefs;
         self.export_prefs = engine_config.export_prefs;
+        self.save_prefs = engine_config.save_prefs.clone_from_engineconfig_to_engine();
 
         // Set the pen sounds to update the audioplayer
         self.set_pen_sounds(engine_config.pen_sounds, data_dir);

crates/rnote-engine/src/engine/mod.rs

@@ -2,6 +2,7 @@
 pub mod export;
 pub mod import;
 pub mod rendering;
+pub mod save;
 pub mod snapshot;
 pub mod strokecontent;
 pub mod visual_debug;
@@ -11,6 +12,7 @@ pub use export::ExportPrefs;
 use futures::StreamExt;
 use futures::channel::mpsc::UnboundedReceiver;
 pub use import::ImportPrefs;
+pub use save::SavePrefs;
 pub use snapshot::EngineSnapshot;
 pub use strokecontent::StrokeContent;
 
@@ -154,6 +156,8 @@ pub struct EngineConfig {
     import_prefs: ImportPrefs,
     #[serde(rename = "export_prefs")]
     export_prefs: ExportPrefs,
+    #[serde(rename = "save_prefs")]
+    save_prefs: SavePrefs,
     #[serde(rename = "pen_sounds")]
     pen_sounds: bool,
     #[serde(rename = "optimize_epd")]
@@ -235,6 +239,8 @@ pub struct Engine {
     pub import_prefs: ImportPrefs,
     #[serde(rename = "export_prefs")]
     pub export_prefs: ExportPrefs,
+    #[serde(rename = "save_prefs")]
+    pub save_prefs: SavePrefs,
     #[serde(rename = "pen_sounds")]
     pen_sounds: bool,
     #[serde(rename = "optimize_epd")]
@@ -277,6 +283,7 @@ impl Default for Engine {
             penholder: PenHolder::default(),
             import_prefs: ImportPrefs::default(),
             export_prefs: ExportPrefs::default(),
+            save_prefs: SavePrefs::default(),
             pen_sounds: false,
             optimize_epd: false,
 
@@ -377,13 +384,16 @@ impl Engine {
             stroke_components: Arc::clone(&store_history_entry.stroke_components),
             chrono_components: Arc::clone(&store_history_entry.chrono_components),
             chrono_counter: store_history_entry.chrono_counter,
+            save_prefs: self.save_prefs.clone_from_engine_to_enginesnapshot(),
         }
     }
 
     /// Imports an engine snapshot. A save file should always be loaded with this method.
     pub fn load_snapshot(&mut self, snapshot: EngineSnapshot) -> WidgetFlags {
         self.document = snapshot.document.clone_config();
         self.camera = snapshot.camera.clone_config();
+        self.save_prefs.merge(&snapshot.save_prefs);
+
         let mut widget_flags = self.store.import_from_snapshot(&snapshot)
             | self.doc_resize_autoexpand()
             | self.current_pen_update_state()