UPSTREAM PR #2375: feat: filters and partial cloning: initial support

gitoxide-core/src/pack/receive.rs

@@ -136,6 +136,7 @@ where
             shallow_file: "no shallow file required as we reject it to keep it simple".into(),
             shallow: &Default::default(),
             tags: Default::default(),
+            filter: None,
             reject_shallow_remote: true,
         },
     )

gitoxide-core/src/repository/clone.rs

@@ -7,6 +7,7 @@ pub struct Options {
     pub no_tags: bool,
     pub shallow: gix::remote::fetch::Shallow,
     pub ref_name: Option<gix::refs::PartialName>,
+    pub filter: Option<gix::remote::fetch::ObjectFilter>,
 }
 
 pub const PROGRESS_RANGE: std::ops::RangeInclusive<u8> = 1..=3;
@@ -34,6 +35,7 @@ pub(crate) mod function {
             no_tags,
             ref_name,
             shallow,
+            filter,
         }: Options,
     ) -> anyhow::Result<()>
     where
@@ -76,6 +78,7 @@ pub(crate) mod function {
             prepare = prepare.configure_remote(|r| Ok(r.with_fetch_tags(gix::remote::fetch::Tags::None)));
         }
         let (mut checkout, fetch_outcome) = prepare
+            .with_filter(filter)
             .with_shallow(shallow)
             .with_ref_name(ref_name.as_ref())?
             .fetch_then_checkout(&mut progress, &gix::interrupt::IS_INTERRUPTED)?;

gitoxide-core/src/repository/fetch.rs

@@ -1,4 +1,4 @@
-use gix::bstr::BString;
+use gix::{bstr::BString, hash::ObjectId};
 
 use crate::OutputFormat;
 
@@ -29,7 +29,7 @@ pub(crate) mod function {
         std_shapes::shapes::{Arrow, Element, ShapeKind},
     };
 
-    use super::Options;
+    use super::{ObjectId, Options};
     use crate::OutputFormat;
 
     pub fn fetch<P>(
@@ -57,15 +57,28 @@ pub(crate) mod function {
         }
 
         let mut remote = crate::repository::remote::by_name_or_url(&repo, remote.as_deref())?;
-        if !ref_specs.is_empty() {
-            remote.replace_refspecs(ref_specs.iter(), gix::remote::Direction::Fetch)?;
+        let mut wants = Vec::new();
+        let mut fetch_refspecs = Vec::new();
+        for spec in ref_specs {
+            if spec.len() == repo.object_hash().len_in_hex() {
+                if let Ok(oid) = ObjectId::from_hex(spec.as_ref()) {
+                    wants.push(oid);
+                    continue;
+                }
+            }
+            fetch_refspecs.push(spec);
+        }
+
+        if !fetch_refspecs.is_empty() {
+            remote.replace_refspecs(fetch_refspecs.iter(), gix::remote::Direction::Fetch)?;
             remote = remote.with_fetch_tags(gix::remote::fetch::Tags::None);
         }
         let res: gix::remote::fetch::Outcome = remote
             .connect(gix::remote::Direction::Fetch)?
             .prepare_fetch(&mut progress, Default::default())?
             .with_dry_run(dry_run)
             .with_shallow(shallow)
+            .with_additional_wants(wants)
             .receive(&mut progress, &gix::interrupt::IS_INTERRUPTED)?;
 
         if handshake_info {

gix-protocol/src/fetch/function.rs

@@ -54,6 +54,7 @@ pub async fn fetch<P, T, E>(
         shallow_file,
         shallow,
         tags,
+        filter,
         reject_shallow_remote,
     }: Options<'_>,
 ) -> Result<Option<Outcome>, Error>
@@ -77,6 +78,15 @@ where
     crate::fetch::Response::check_required_features(protocol_version, &fetch_features)?;
     let sideband_all = fetch_features.iter().any(|(n, _)| *n == "sideband-all");
     let mut arguments = Arguments::new(protocol_version, fetch_features, trace_packetlines);
+    if let Some(filter) = filter {
+        if !arguments.can_use_filter() {
+            return Err(Error::MissingServerFeature {
+                feature: "filter",
+                description: "Partial clone filters require server support configured on the remote server",
+            });
+        }
+        arguments.filter(filter);
+    }
     if matches!(tags, Tags::Included) {
         if !arguments.can_use_include_tag() {
             return Err(Error::MissingServerFeature {

gix-protocol/src/fetch/types.rs

@@ -13,6 +13,8 @@ pub struct Options<'a> {
     pub shallow: &'a Shallow,
     /// Describe how to handle tags when fetching.
     pub tags: Tags,
+    /// If set, request that the remote filters the object set according to `filter`.
+    pub filter: Option<&'a str>,
     /// If `true`, if we fetch from a remote that only offers shallow clones, the operation will fail with an error
     /// instead of writing the shallow boundary to the shallow file.
     pub reject_shallow_remote: bool,

gix/src/clone/access.rs

@@ -35,6 +35,12 @@ impl PrepareFetch {
         self
     }
 
+    /// Ask the remote to omit objects based on `filter`.
+    pub fn with_filter(mut self, filter: impl Into<Option<crate::remote::fetch::ObjectFilter>>) -> Self {
+        self.filter = filter.into();
+        self
+    }
+
     /// Apply the given configuration `values` right before readying the actual fetch from the remote.
     /// The configuration is marked with [source API](gix_config::Source::Api), and will not be written back, it's
     /// retained only in memory.

gix/src/clone/fetch/mod.rs

@@ -20,6 +20,13 @@ pub enum Error {
     RemoteConfiguration(#[source] Box<dyn std::error::Error + Send + Sync>),
     #[error("Custom configuration of connection to use when cloning failed")]
     RemoteConnection(#[source] Box<dyn std::error::Error + Send + Sync>),
+    #[error("Remote name {remote_name:?} is not valid UTF-8")]
+    RemoteNameNotUtf8 {
+        source: crate::bstr::Utf8Error,
+        remote_name: crate::bstr::BString,
+    },
+    #[error(transparent)]
+    ConfigSectionHeader(#[from] gix_config::parse::section::header::Error),
     #[error(transparent)]
     RemoteName(#[from] crate::config::remote::symbolic_name::Error),
     #[error(transparent)]
@@ -203,7 +210,12 @@ impl PrepareFetch {
             clone_fetch_tags = remote::fetch::Tags::All.into();
         }
 
-        let config = util::write_remote_to_local_config_file(&mut remote, remote_name.clone())?;
+        let filter_spec_to_save = self
+            .filter
+            .as_ref()
+            .map(remote::fetch::ObjectFilter::to_argument_string);
+        let config =
+            util::write_remote_to_local_config_file(&mut remote, remote_name.clone(), filter_spec_to_save.as_deref())?;
 
         // Now we are free to apply remote configuration we don't want to be written to disk.
         if let Some(fetch_tags) = clone_fetch_tags {
@@ -283,6 +295,7 @@ impl PrepareFetch {
             b
         };
         let outcome = pending_pack
+            .with_filter(self.filter.clone())
             .with_write_packed_refs_only(true)
             .with_reflog_message(RefLogMessage::Override {
                 message: reflog_message.clone(),
@@ -326,3 +339,5 @@ impl PrepareFetch {
 }
 
 mod util;
+
+pub use util::write_remote_to_local_config_file;

gix/src/clone/fetch/util.rs

@@ -16,13 +16,40 @@ enum WriteMode {
     Append,
 }
 
+/// Persist the provided remote into the repository's local config, optionally adding partial clone settings.
 #[allow(clippy::result_large_err)]
 pub fn write_remote_to_local_config_file(
     remote: &mut crate::Remote<'_>,
     remote_name: BString,
+    filter: Option<&str>,
 ) -> Result<gix_config::File<'static>, Error> {
+    use gix_config::parse::section::ValueName;
+
     let mut config = gix_config::File::new(local_config_meta(remote.repo));
-    remote.save_as_to(remote_name, &mut config)?;
+    remote.save_as_to(remote_name.clone(), &mut config)?;
+
+    if let Some(filter_spec) = filter {
+        let subsection = remote_name.to_str().map_err(|err| Error::RemoteNameNotUtf8 {
+            remote_name: remote_name.clone(),
+            source: err,
+        })?;
+        let mut remote_section = config.section_mut_or_create_new("remote", Some(subsection.into()))?;
+
+        while remote_section.remove("partialclonefilter").is_some() {}
+        while remote_section.remove("promisor").is_some() {}
+
+        let partial_clone_filter = ValueName::try_from("partialclonefilter").expect("known to be valid");
+        remote_section.push(partial_clone_filter, Some(filter_spec.into()));
+
+        let promisor = ValueName::try_from("promisor").expect("known to be valid");
+        remote_section.push(promisor, Some("true".into()));
+
+        let mut extensions_section = config.section_mut_or_create_new("extensions", None)?;
+        while extensions_section.remove("partialClone").is_some() {}
+
+        let partial_clone = ValueName::try_from("partialClone").expect("known to be valid");
+        extensions_section.push(partial_clone, Some(remote_name.as_ref()));
+    }
 
     write_to_local_config(&config, WriteMode::Append)?;
     Ok(config)

gix/src/clone/mod.rs

@@ -39,6 +39,9 @@ pub struct PrepareFetch {
     /// How to handle shallow clones
     #[cfg_attr(not(feature = "blocking-network-client"), allow(dead_code))]
     shallow: remote::fetch::Shallow,
+    /// Optional object filter to request from the remote.
+    #[cfg_attr(not(feature = "blocking-network-client"), allow(dead_code))]
+    filter: Option<remote::fetch::ObjectFilter>,
     /// The name of the reference to fetch. If `None`, the reference pointed to by `HEAD` will be checked out.
     #[cfg_attr(not(feature = "blocking-network-client"), allow(dead_code))]
     ref_name: Option<gix_ref::PartialName>,
@@ -121,6 +124,7 @@ impl PrepareFetch {
             #[cfg(any(feature = "async-network-client", feature = "blocking-network-client"))]
             configure_connection: None,
             shallow: remote::fetch::Shallow::NoChange,
+            filter: None,
             ref_name: None,
         })
     }

gix/src/remote/connection/fetch/mod.rs

@@ -12,6 +12,7 @@ use crate::{
     },
     Progress,
 };
+use gix_hash::ObjectId;
 
 mod error;
 pub use error::Error;
@@ -153,6 +154,8 @@ where
             reflog_message: None,
             write_packed_refs: WritePackedRefs::Never,
             shallow: Default::default(),
+            filter: None,
+            additional_wants: Vec::new(),
         })
     }
 }
@@ -184,6 +187,8 @@ where
     reflog_message: Option<RefLogMessage>,
     write_packed_refs: WritePackedRefs,
     shallow: remote::fetch::Shallow,
+    filter: Option<remote::fetch::ObjectFilter>,
+    additional_wants: Vec<ObjectId>,
 }
 
 /// Builder
@@ -225,4 +230,22 @@ where
         self.shallow = shallow;
         self
     }
+
+    /// Ask the server to apply `filter` when sending objects.
+    pub fn with_filter(mut self, filter: Option<remote::fetch::ObjectFilter>) -> Self {
+        self.filter = filter;
+        self
+    }
+
+    /// Request that the server also sends the objects identified by the given object ids.
+    ///
+    /// Objects already present locally will be ignored during negotiation.
+    pub fn with_additional_wants(mut self, wants: impl IntoIterator<Item = ObjectId>) -> Self {
+        for want in wants {
+            if !self.additional_wants.contains(&want) {
+                self.additional_wants.push(want);
+            }
+        }
+        self
+    }
 }