refactor: store TableSchema partition columns as Fields

Store `TableSchema::table_partition_cols` as `arrow::datatypes::Fields`
(an immutable `Arc<[FieldRef]>`) instead of `Arc<Vec<FieldRef>>`.

Now that `with_table_partition_cols` replaces rather than appends, the
inner `Vec` only existed to support in-place `extend`. `Fields` is the
idiomatic Arrow type for an immutable field list: it is a single
`Arc<[FieldRef]>` (one fewer indirection than `Arc<Vec<_>>`), can be
shared zero-copy with an existing schema, and makes the shared-`Arc`
mutation panic that motivated the replace change structurally impossible
-- there is no `Vec` to `Arc::get_mut`/`make_mut`.

`TableSchema::table_partition_cols()` and the delegating
`FileScanConfig::table_partition_cols()` now return `&Fields`. `Fields`
derefs to `&[FieldRef]`, so iteration/indexing/`len`/`is_empty` callers
are unchanged; only the arrow `FileFormat` path needed `.to_vec()` to
rebuild an owned `Vec`. The constructors still take `Vec<FieldRef>`, so
no call site that builds partition columns changes.

Documented the `&Fields` return-type change in the 54.0.0 upgrade guide.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: adriangb

datafusion/datasource-arrow/src/file_format.rs

@@ -199,7 +199,7 @@ impl FileFormat for ArrowFormat {
 
         let table_schema = TableSchema::new(
             Arc::clone(conf.file_schema()),
-            conf.table_partition_cols().clone(),
+            conf.table_partition_cols().to_vec(),
         );
 
         let mut source: Arc<dyn FileSource> =

datafusion/datasource/src/file_scan_config/mod.rs

@@ -27,7 +27,7 @@ use crate::{
     file_stream::work_source::SharedWorkSource, source::DataSource,
     statistics::MinMaxStatistics,
 };
-use arrow::datatypes::FieldRef;
+use arrow::datatypes::Fields;
 use arrow::datatypes::{DataType, Schema, SchemaRef};
 use datafusion_common::config::ConfigOptions;
 use datafusion_common::tree_node::TreeNodeRecursion;
@@ -1068,7 +1068,7 @@ impl FileScanConfig {
     }
 
     /// Get the table partition columns
-    pub fn table_partition_cols(&self) -> &Vec<FieldRef> {
+    pub fn table_partition_cols(&self) -> &Fields {
         self.file_source.table_schema().table_partition_cols()
     }
 
@@ -2074,7 +2074,10 @@ mod tests {
             Some(vec![0, 2])
         );
         assert_eq!(new_config.limit, Some(10));
-        assert_eq!(*new_config.table_partition_cols(), partition_cols);
+        assert_eq!(
+            *new_config.table_partition_cols(),
+            Fields::from(partition_cols)
+        );
         assert_eq!(new_config.file_groups.len(), 1);
         assert_eq!(new_config.file_groups[0].len(), 1);
         assert_eq!(

datafusion/datasource/src/table_schema.rs

@@ -17,7 +17,7 @@
 
 //! Helper struct to manage table schemas with partition columns
 
-use arrow::datatypes::{FieldRef, SchemaBuilder, SchemaRef};
+use arrow::datatypes::{FieldRef, Fields, SchemaBuilder, SchemaRef};
 use std::sync::Arc;
 
 /// The overall schema for potentially partitioned data sources.
@@ -70,7 +70,11 @@ pub struct TableSchema {
     ///
     /// These columns are NOT present in the data files but are appended to each
     /// row during query execution based on the file's location.
-    table_partition_cols: Arc<Vec<FieldRef>>,
+    ///
+    /// Stored as [`Fields`] (an immutable `Arc<[FieldRef]>`) so that cloning a
+    /// `TableSchema` is cheap and the partition columns can be shared zero-copy
+    /// with an existing schema.
+    table_partition_cols: Fields,
 
     /// The complete table schema: file_schema columns followed by partition columns.
     ///
@@ -117,11 +121,14 @@ impl TableSchema {
     /// assert_eq!(table_schema.table_schema().fields().len(), 4);
     /// ```
     pub fn new(file_schema: SchemaRef, table_partition_cols: Vec<FieldRef>) -> Self {
+        // Stored immutably as `Fields`; cloning a `TableSchema` is then a cheap
+        // refcount bump and the partition columns can never be mutated in place.
+        let table_partition_cols = Fields::from(table_partition_cols);
         let mut builder = SchemaBuilder::from(file_schema.as_ref());
         builder.extend(table_partition_cols.iter().cloned());
         Self {
             file_schema,
-            table_partition_cols: Arc::new(table_partition_cols),
+            table_partition_cols,
             table_schema: Arc::new(builder.finish()),
         }
     }
@@ -144,13 +151,13 @@ impl TableSchema {
     /// into [`TableSchema::with_table_partition_cols`] if you have partition columns at construction time
     /// since it avoids re-computing the table schema.
     pub fn with_table_partition_cols(mut self, partition_cols: Vec<FieldRef>) -> Self {
-        // Replace the partition columns outright. Assigning a fresh `Arc` is
-        // cheap and never mutates the inner `Vec`, so this is safe even when the
-        // `Arc` is shared with a clone of this `TableSchema` (copy-on-write
-        // isolation is automatic). The previous `Arc::get_mut().expect()`
-        // appended in place and panicked whenever the `Arc` was shared, since
-        // owning `self` does not imply sole ownership of the inner `Arc`.
-        self.table_partition_cols = Arc::new(partition_cols);
+        // Replace the partition columns outright. `Fields` is an immutable
+        // `Arc<[FieldRef]>`, so storing it never mutates a shared allocation and
+        // is safe even when this `TableSchema` has been cloned. (A previous
+        // `Arc<Vec<_>>` design appended in place via `Arc::get_mut().expect()`
+        // and panicked whenever the `Arc` was shared, since owning `self` does
+        // not imply sole ownership of the inner `Arc`.)
+        self.table_partition_cols = Fields::from(partition_cols);
         let mut builder = SchemaBuilder::from(self.file_schema.as_ref());
         builder.extend(self.table_partition_cols.iter().cloned());
         self.table_schema = Arc::new(builder.finish());
@@ -168,7 +175,7 @@ impl TableSchema {
     ///
     /// These are the columns derived from the directory structure that
     /// will be appended to each row during query execution.
-    pub fn table_partition_cols(&self) -> &Vec<FieldRef> {
+    pub fn table_partition_cols(&self) -> &Fields {
         &self.table_partition_cols
     }
 

docs/source/library-user-guide/upgrading/54.0.0.md

@@ -787,3 +787,31 @@ a combined `OR` (e.g., index-backed sources).
 ```sql
 SET datafusion.optimizer.enable_unions_to_filter = true;
 ```
+
+### `TableSchema::table_partition_cols()` now returns `&Fields`
+
+`datafusion_datasource::TableSchema` now stores its partition columns as
+`arrow::datatypes::Fields` (an immutable `Arc<[FieldRef]>`) instead of
+`Arc<Vec<FieldRef>>`. As a result:
+
+- `TableSchema::table_partition_cols()` now returns `&Fields` instead of
+  `&Vec<FieldRef>`.
+- `FileScanConfig::table_partition_cols()` (which delegates to it) likewise now
+  returns `&Fields`.
+
+`Fields` dereferences to `&[FieldRef]`, so the common operations — iteration,
+indexing, `len()`, `is_empty()` — are unchanged. Only code that named the
+concrete `&Vec<FieldRef>` type (e.g. binding the result to a `&Vec<FieldRef>` or
+calling `Vec`-specific methods) needs to adapt:
+
+```rust
+// Before: returned &Vec<FieldRef>
+let cols: Vec<FieldRef> = config.table_partition_cols().clone();
+
+// After: returns &Fields; use `to_vec()` when you need an owned `Vec`
+let cols: Vec<FieldRef> = config.table_partition_cols().to_vec();
+```
+
+The constructors `TableSchema::new` and `TableSchema::with_table_partition_cols`
+still accept `Vec<FieldRef>`, so code that builds partition columns is
+unaffected.