diff --git a/Cargo.lock b/Cargo.lock index 46cf27d959..a7a6bed67e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2916,7 +2916,7 @@ dependencies = [ "brotli", "candid", "derivative", - "dfx-core", + "dunce", "flate2", "futures", "futures-intrusive", diff --git a/Cargo.toml b/Cargo.toml index b322d2efbb..3a1ab82bc1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ clap = "4.5" clap_complete = "4.5" dialoguer = "0.11.0" directories-next = "2.0.0" +dunce = "1.0" flate2 = { version = "1.0.11" } futures = "0.3.21" handlebars = "4.3.3" @@ -64,9 +65,7 @@ mime_guess = "2.0.4" num-traits = "0.2.14" pem = "1.0.2" proptest = "1.0.0" -reqwest = { version = "0.12.4", default-features = false, features = [ - "rustls-tls", -] } +reqwest = { version = "0.12.4", default-features = false, features = ["rustls-tls"] } ring = { version = "0.17.14", features = ["std"] } schemars = "0.8" sec1 = "0.3.0" diff --git a/src/canisters/frontend/ic-asset/Cargo.toml b/src/canisters/frontend/ic-asset/Cargo.toml index db2053922d..9d8bafdbf3 100644 --- a/src/canisters/frontend/ic-asset/Cargo.toml +++ b/src/canisters/frontend/ic-asset/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ic-asset" -version = "0.25.0" # sync with icx-asset +version = "0.25.0" # sync with icx-asset authors.workspace = true edition.workspace = true repository.workspace = true @@ -19,7 +19,7 @@ backoff.workspace = true brotli = "6.0.0" candid = { workspace = true } derivative = "2.2.0" -dfx-core.workspace = true +dunce.workspace = true flate2.workspace = true futures.workspace = true futures-intrusive = "0.4.0" diff --git a/src/canisters/frontend/ic-asset/src/asset/config.rs b/src/canisters/frontend/ic-asset/src/asset/config.rs index 4bd370b67d..9c321369f9 100644 --- a/src/canisters/frontend/ic-asset/src/asset/config.rs +++ b/src/canisters/frontend/ic-asset/src/asset/config.rs @@ -176,7 +176,7 @@ impl AssetSourceDirectoryConfiguration { &mut self, canonical_path: &Path, ) -> Result { - let parent_dir = dfx_core::fs::parent(canonical_path)?; + let parent_dir = crate::fs::parent(canonical_path)?; Ok(self .config_map .get(&parent_dir) @@ -243,7 +243,7 @@ impl AssetConfigTreeNode { }; let mut rules = vec![]; if let Some(config_path) = config_path { - let content = dfx_core::fs::read_to_string(&config_path)?; + let content = crate::fs::read_to_string(&config_path)?; let interim_rules: Vec = json5::from_str(&content) .map_err(|e| MalformedAssetConfigFile(config_path.to_path_buf(), e))?; @@ -264,7 +264,7 @@ impl AssetConfigTreeNode { }; configs.insert(dir.to_path_buf(), parent_ref.clone()); - for f in dfx_core::fs::read_dir(dir)? + for f in crate::fs::read_dir(dir)? .filter_map(|x| x.ok()) .filter(|x| x.file_type().map_or_else(|_e| false, |ft| ft.is_dir())) { diff --git a/src/canisters/frontend/ic-asset/src/asset/content.rs b/src/canisters/frontend/ic-asset/src/asset/content.rs index e8276f5f00..f616f944b8 100644 --- a/src/canisters/frontend/ic-asset/src/asset/content.rs +++ b/src/canisters/frontend/ic-asset/src/asset/content.rs @@ -1,6 +1,6 @@ use crate::asset::content_encoder::ContentEncoder; +use crate::error::fs::ReadFileError; use brotli::CompressorWriter; -use dfx_core::error::fs::ReadFileError; use flate2::Compression; use flate2::write::GzEncoder; use mime::Mime; @@ -16,7 +16,7 @@ pub(crate) struct Content { impl Content { pub fn load(path: &Path) -> Result { - let data = dfx_core::fs::read(path)?; + let data = crate::fs::read(path)?; // todo: check contents if mime_guess fails https://github.com/dfinity/sdk/issues/1594 let media_type = mime_guess::from_path(path) diff --git a/src/canisters/frontend/ic-asset/src/batch_upload/plumbing.rs b/src/canisters/frontend/ic-asset/src/batch_upload/plumbing.rs index 15ab592632..b2a57caa38 100644 --- a/src/canisters/frontend/ic-asset/src/batch_upload/plumbing.rs +++ b/src/canisters/frontend/ic-asset/src/batch_upload/plumbing.rs @@ -448,7 +448,7 @@ async fn make_project_asset( logger: &Logger, progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result { - let file_size = dfx_core::fs::metadata(&asset_descriptor.source)?.len(); + let file_size = crate::fs::metadata(&asset_descriptor.source)?.len(); let permits = (file_size.div_ceil(1000000) as usize).clamp(1, MAX_COST_SINGLE_FILE_MB); let _releaser = semaphores.file.acquire(permits).await; let content = Content::load(&asset_descriptor.source) diff --git a/src/canisters/frontend/ic-asset/src/error/create_project_asset.rs b/src/canisters/frontend/ic-asset/src/error/create_project_asset.rs index e14f9bea69..341a6130f7 100644 --- a/src/canisters/frontend/ic-asset/src/error/create_project_asset.rs +++ b/src/canisters/frontend/ic-asset/src/error/create_project_asset.rs @@ -1,5 +1,5 @@ use crate::error::create_encoding::CreateEncodingError; -use dfx_core::error::fs::{ReadFileError, ReadMetadataError}; +use crate::error::fs::{ReadFileError, ReadMetadataError}; use thiserror::Error; /// Errors related to creating an asset found in the project in the asset canister. diff --git a/src/canisters/frontend/ic-asset/src/error/fs.rs b/src/canisters/frontend/ic-asset/src/error/fs.rs new file mode 100644 index 0000000000..c07952a353 --- /dev/null +++ b/src/canisters/frontend/ic-asset/src/error/fs.rs @@ -0,0 +1,133 @@ +use std::path::PathBuf; +use thiserror::Error; + +#[derive(Error, Debug)] +#[error("failed to canonicalize '{path}'")] +pub struct CanonicalizePathError { + pub path: PathBuf, + pub source: std::io::Error, +} + +#[derive(Error, Debug)] +#[error("failed to copy {from} to {to}")] +pub struct CopyFileError { + pub from: PathBuf, + pub to: PathBuf, + pub source: std::io::Error, +} + +#[derive(Error, Debug)] +#[error("failed to create directory {path} and parents")] +pub struct CreateDirAllError { + pub path: PathBuf, + pub source: std::io::Error, +} + +#[derive(Error, Debug)] +pub enum EnsureDirExistsError { + #[error(transparent)] + CreateDirAll(#[from] CreateDirAllError), + + #[error("path {0} is not a directory")] + NotADirectory(PathBuf), +} + +#[derive(Error, Debug)] +pub enum EnsureParentDirExistsError { + #[error(transparent)] + EnsureDirExists(#[from] EnsureDirExistsError), + + #[error(transparent)] + NoParentPath(#[from] NoParentPathError), +} + +#[derive(Error, Debug)] +#[error("failed to determine parent path for '{0}'")] +pub struct NoParentPathError(pub PathBuf); + +#[derive(Error, Debug)] +#[error("failed to read directory {path}")] +pub struct ReadDirError { + pub path: PathBuf, + pub source: std::io::Error, +} + +#[derive(Error, Debug)] +#[error("failed to read from {path}")] +pub struct ReadFileError { + pub path: PathBuf, + pub source: std::io::Error, +} + +#[derive(Error, Debug)] +#[error("failed to remove directory {path}")] +pub struct RemoveDirectoryError { + pub path: PathBuf, + pub source: std::io::Error, +} + +#[derive(Error, Debug)] +#[error("failed to write to {path}")] +pub struct WriteFileError { + pub path: PathBuf, + pub source: std::io::Error, +} + +#[derive(Error, Debug)] +#[error("failed to read metadata of {path}")] +pub struct ReadMetadataError { + pub path: PathBuf, + pub source: std::io::Error, +} + +#[derive(Error, Debug)] +#[error("failed to read permissions of {path}")] +pub struct ReadPermissionsError { + pub path: PathBuf, + pub source: std::io::Error, +} + +#[derive(Error, Debug)] +#[error("failed to read {path} as string")] +pub struct ReadToStringError { + pub path: PathBuf, + pub source: std::io::Error, +} + +#[derive(Error, Debug)] +#[error("failed to remove directory {path} and its contents")] +pub struct RemoveDirectoryAndContentsError { + pub path: PathBuf, + pub source: std::io::Error, +} + +#[derive(Error, Debug)] +#[error("failed to remove file {path}")] +pub struct RemoveFileError { + pub path: PathBuf, + pub source: std::io::Error, +} + +#[derive(Error, Debug)] +#[error("Failed to rename {from} to {to}")] +pub struct RenameError { + pub from: PathBuf, + pub to: PathBuf, + pub source: std::io::Error, +} + +#[derive(Error, Debug)] +#[error("failed to set permissions of {path}")] +pub struct SetPermissionsError { + pub path: PathBuf, + pub source: std::io::Error, +} + +#[derive(Error, Debug)] +pub enum SetPermissionsReadWriteError { + #[error(transparent)] + ReadPermissions(#[from] ReadPermissionsError), + + #[error(transparent)] + SetPermissions(#[from] SetPermissionsError), +} diff --git a/src/canisters/frontend/ic-asset/src/error/gather_asset_descriptors.rs b/src/canisters/frontend/ic-asset/src/error/gather_asset_descriptors.rs index 532ec4359b..1ea3c3f8c8 100644 --- a/src/canisters/frontend/ic-asset/src/error/gather_asset_descriptors.rs +++ b/src/canisters/frontend/ic-asset/src/error/gather_asset_descriptors.rs @@ -1,6 +1,6 @@ +use crate::error::fs::CanonicalizePathError; use crate::error::get_asset_config::GetAssetConfigError; use crate::error::load_config::AssetLoadConfigError; -use dfx_core::error::fs::CanonicalizePathError; use std::path::PathBuf; use thiserror::Error; diff --git a/src/canisters/frontend/ic-asset/src/error/get_asset_config.rs b/src/canisters/frontend/ic-asset/src/error/get_asset_config.rs index 4e87d09bdf..1e96d44fd2 100644 --- a/src/canisters/frontend/ic-asset/src/error/get_asset_config.rs +++ b/src/canisters/frontend/ic-asset/src/error/get_asset_config.rs @@ -1,4 +1,4 @@ -use dfx_core::error::fs::NoParentPathError; +use crate::error::fs::NoParentPathError; use std::path::PathBuf; use thiserror::Error; diff --git a/src/canisters/frontend/ic-asset/src/error/hash_content.rs b/src/canisters/frontend/ic-asset/src/error/hash_content.rs index 0bdcd138b9..684fe1c38e 100644 --- a/src/canisters/frontend/ic-asset/src/error/hash_content.rs +++ b/src/canisters/frontend/ic-asset/src/error/hash_content.rs @@ -1,5 +1,5 @@ use crate::asset::content_encoder::ContentEncoder; -use dfx_core::error::fs::ReadFileError; +use crate::error::fs::ReadFileError; use thiserror::Error; /// Errors related to hashing asset content. diff --git a/src/canisters/frontend/ic-asset/src/error/load_config.rs b/src/canisters/frontend/ic-asset/src/error/load_config.rs index 40410b4625..59d1e57c66 100644 --- a/src/canisters/frontend/ic-asset/src/error/load_config.rs +++ b/src/canisters/frontend/ic-asset/src/error/load_config.rs @@ -1,5 +1,5 @@ +use crate::error::fs::{ReadDirError, ReadToStringError}; use crate::error::load_rule::LoadRuleError; -use dfx_core::error::fs::{ReadDirError, ReadToStringError}; use std::path::PathBuf; use thiserror::Error; diff --git a/src/canisters/frontend/ic-asset/src/error/mod.rs b/src/canisters/frontend/ic-asset/src/error/mod.rs index bd2ac50c35..81e155476d 100644 --- a/src/canisters/frontend/ic-asset/src/error/mod.rs +++ b/src/canisters/frontend/ic-asset/src/error/mod.rs @@ -7,6 +7,7 @@ mod create_chunk; mod create_encoding; mod create_project_asset; mod downgrade_commit_batch_arguments; +pub(crate) mod fs; mod gather_asset_descriptors; mod get_asset_config; mod get_asset_properties; diff --git a/src/canisters/frontend/ic-asset/src/fs/composite.rs b/src/canisters/frontend/ic-asset/src/fs/composite.rs new file mode 100644 index 0000000000..f6ff649453 --- /dev/null +++ b/src/canisters/frontend/ic-asset/src/fs/composite.rs @@ -0,0 +1,20 @@ +use crate::error::fs::EnsureDirExistsError::NotADirectory; +use crate::error::fs::{EnsureDirExistsError, EnsureParentDirExistsError}; +use std::path::Path; + +pub fn ensure_dir_exists(p: &Path) -> Result<(), EnsureDirExistsError> { + if !p.exists() { + crate::fs::create_dir_all(p)?; + Ok(()) + } else if !p.is_dir() { + Err(NotADirectory(p.to_path_buf())) + } else { + Ok(()) + } +} + +pub fn ensure_parent_dir_exists(d: &Path) -> Result<(), EnsureParentDirExistsError> { + let parent = crate::fs::parent(d)?; + ensure_dir_exists(&parent)?; + Ok(()) +} diff --git a/src/canisters/frontend/ic-asset/src/fs/mod.rs b/src/canisters/frontend/ic-asset/src/fs/mod.rs new file mode 100644 index 0000000000..a14912ed06 --- /dev/null +++ b/src/canisters/frontend/ic-asset/src/fs/mod.rs @@ -0,0 +1,131 @@ +#![allow(unused)] // Copied from dfx-core +pub mod composite; +use crate::error::fs::{ + CanonicalizePathError, CopyFileError, CreateDirAllError, NoParentPathError, ReadDirError, + ReadFileError, ReadMetadataError, ReadPermissionsError, ReadToStringError, + RemoveDirectoryAndContentsError, RemoveDirectoryError, RemoveFileError, RenameError, + SetPermissionsError, SetPermissionsReadWriteError, WriteFileError, +}; +use std::fs::{Metadata, Permissions, ReadDir}; +use std::path::{Path, PathBuf}; + +pub fn canonicalize(path: &Path) -> Result { + dunce::canonicalize(path).map_err(|source| CanonicalizePathError { + path: path.to_path_buf(), + source, + }) +} + +pub fn copy(from: &Path, to: &Path) -> Result { + std::fs::copy(from, to).map_err(|source| CopyFileError { + from: from.to_path_buf(), + to: to.to_path_buf(), + source, + }) +} + +pub fn create_dir_all(path: &Path) -> Result<(), CreateDirAllError> { + std::fs::create_dir_all(path).map_err(|source| CreateDirAllError { + path: path.to_path_buf(), + source, + }) +} + +pub fn metadata(path: &Path) -> Result { + std::fs::metadata(path).map_err(|source| ReadMetadataError { + path: path.to_path_buf(), + source, + }) +} + +pub fn parent(path: &Path) -> Result { + match path.parent() { + None => Err(NoParentPathError(path.to_path_buf())), + Some(parent) => Ok(parent.to_path_buf()), + } +} + +pub fn read(path: &Path) -> Result, ReadFileError> { + std::fs::read(path).map_err(|source| ReadFileError { + path: path.to_path_buf(), + source, + }) +} + +pub fn read_to_string(path: &Path) -> Result { + std::fs::read_to_string(path).map_err(|source| ReadToStringError { + path: path.to_path_buf(), + source, + }) +} + +pub fn read_dir(path: &Path) -> Result { + path.read_dir().map_err(|source| ReadDirError { + path: path.to_path_buf(), + source, + }) +} + +pub fn rename(from: &Path, to: &Path) -> Result<(), RenameError> { + std::fs::rename(from, to).map_err(|source| RenameError { + from: from.to_path_buf(), + to: to.to_path_buf(), + source, + }) +} + +pub fn read_permissions(path: &Path) -> Result { + std::fs::metadata(path) + .map_err(|source| ReadPermissionsError { + path: path.to_path_buf(), + source, + }) + .map(|x| x.permissions()) +} + +pub fn remove_dir(path: &Path) -> Result<(), RemoveDirectoryError> { + std::fs::remove_dir(path).map_err(|source| RemoveDirectoryError { + path: path.to_path_buf(), + source, + }) +} + +pub fn remove_dir_all(path: &Path) -> Result<(), RemoveDirectoryAndContentsError> { + std::fs::remove_dir_all(path).map_err(|source| RemoveDirectoryAndContentsError { + path: path.to_path_buf(), + source, + }) +} + +pub fn remove_file(path: &Path) -> Result<(), RemoveFileError> { + std::fs::remove_file(path).map_err(|source| RemoveFileError { + path: path.to_path_buf(), + source, + }) +} + +pub fn set_permissions(path: &Path, permissions: Permissions) -> Result<(), SetPermissionsError> { + std::fs::set_permissions(path, permissions).map_err(|source| SetPermissionsError { + path: path.to_path_buf(), + source, + }) +} + +#[cfg_attr(not(unix), allow(unused_variables))] +pub fn set_permissions_readwrite(path: &Path) -> Result<(), SetPermissionsReadWriteError> { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut permissions = read_permissions(path)?; + permissions.set_mode(permissions.mode() | 0o600); + set_permissions(path, permissions)?; + } + Ok(()) +} + +pub fn write, C: AsRef<[u8]>>(path: P, contents: C) -> Result<(), WriteFileError> { + std::fs::write(path.as_ref(), contents).map_err(|source| WriteFileError { + path: path.as_ref().to_path_buf(), + source, + }) +} diff --git a/src/canisters/frontend/ic-asset/src/lib.rs b/src/canisters/frontend/ic-asset/src/lib.rs index 70c3b68e72..02b5fff372 100644 --- a/src/canisters/frontend/ic-asset/src/lib.rs +++ b/src/canisters/frontend/ic-asset/src/lib.rs @@ -36,6 +36,7 @@ mod batch_upload; mod canister_api; pub mod error; mod evidence; +mod fs; mod progress; pub mod security_policy; mod sync; diff --git a/src/canisters/frontend/ic-asset/src/sync.rs b/src/canisters/frontend/ic-asset/src/sync.rs index 3ac275a0b5..c331695ded 100644 --- a/src/canisters/frontend/ic-asset/src/sync.rs +++ b/src/canisters/frontend/ic-asset/src/sync.rs @@ -398,14 +398,14 @@ pub(crate) fn gather_asset_descriptors( ) -> Result, GatherAssetDescriptorsError> { let mut asset_descriptors: HashMap = HashMap::new(); for dir in dirs { - let dir = dfx_core::fs::canonicalize(dir).map_err(InvalidSourceDirectory)?; + let dir = crate::fs::canonicalize(dir).map_err(InvalidSourceDirectory)?; let mut configuration = AssetSourceDirectoryConfiguration::load(&dir).map_err(LoadConfigFailed)?; let mut asset_descriptors_interim = vec![]; let entries = WalkDir::new(&dir) .into_iter() .filter_entry(|entry| { - if let Ok(canonical_path) = &dfx_core::fs::canonicalize(entry.path()) { + if let Ok(canonical_path) = &crate::fs::canonicalize(entry.path()) { let config = configuration .get_asset_config(canonical_path) .unwrap_or_default(); @@ -421,7 +421,7 @@ pub(crate) fn gather_asset_descriptors( .collect::>(); for e in entries { - let source = dfx_core::fs::canonicalize(e.path()).map_err(InvalidDirectoryEntry)?; + let source = crate::fs::canonicalize(e.path()).map_err(InvalidDirectoryEntry)?; let relative = source.strip_prefix(&dir).expect("cannot strip prefix"); let key = String::from("/") + relative.to_string_lossy().as_ref(); let config = configuration.get_asset_config(&source)?; diff --git a/src/dfx-core/Cargo.toml b/src/dfx-core/Cargo.toml index 74e943c5d1..be62eeaf18 100644 --- a/src/dfx-core/Cargo.toml +++ b/src/dfx-core/Cargo.toml @@ -26,7 +26,7 @@ candid = { workspace = true } clap = { workspace = true, features = ["string", "derive"] } dialoguer = { workspace = true } directories-next.workspace = true -dunce = "1.0" +dunce = { workspace = true } flate2 = { workspace = true } handlebars.workspace = true hex = { workspace = true, features = ["serde"] }