feat(libstore): support S3 Transfer Acceleration

doc/manual/rl-next/s3-transfer-acceleration.md

@@ -0,0 +1,30 @@
+---
+synopsis: "S3 Transfer Acceleration support for faster remote cache access"
+issues: [12973]
+prs: [14277]
+---
+
+S3 binary cache stores now support AWS S3 Transfer Acceleration for faster
+uploads and downloads when accessing buckets from geographically distant
+locations.
+
+Transfer Acceleration routes S3 requests through CloudFront edge locations,
+which can significantly improve performance for users far from the bucket's
+region. For example, US-based users accessing Tokyo-region buckets can see
+substantial speed improvements.
+
+To enable transfer acceleration, add `use-transfer-acceleration=true` to your
+S3 URL:
+
+```console
+$ nix copy nixpkgs.hello \
+  --to 's3://my-cache?region=ap-northeast-1&use-transfer-acceleration=true'
+```
+
+Requirements:
+- Bucket names cannot contain dots (periods)
+- Transfer Acceleration must be enabled on the S3 bucket
+- Additional AWS charges apply for Transfer Acceleration
+
+This feature only applies to AWS S3 buckets and has no effect when using custom
+endpoints for S3-compatible services.

src/libstore-tests/s3-binary-cache-store.cc

@@ -140,4 +140,70 @@ TEST(S3BinaryCacheStore, storageClassConfiguration)
     EXPECT_EQ(config.storageClass.get(), std::optional<std::string>("GLACIER"));
 }
 
+#if NIX_WITH_AWS_AUTH
+/**
+ * Test that transfer acceleration parameter is properly preserved
+ */
+TEST(S3BinaryCacheStore, transferAccelerationParameter)
+{
+    StringMap params;
+    params["use-transfer-acceleration"] = "true";
+
+    S3BinaryCacheStoreConfig config("s3", "my-cache", params);
+
+    // Transfer acceleration param should be in cacheUri.query
+    EXPECT_EQ(
+        config.cacheUri,
+        (ParsedURL{
+            .scheme = "s3",
+            .authority = ParsedURL::Authority{.host = "my-cache"},
+            .query = (StringMap) {{"use-transfer-acceleration", "true"}},
+        }));
+
+    // And the config setting should be set
+    EXPECT_EQ(config.use_transfer_acceleration.get(), true);
+}
+
+/**
+ * Test that transfer acceleration works with other S3 parameters
+ */
+TEST(S3BinaryCacheStore, transferAccelerationWithOtherParams)
+{
+    StringMap params;
+    params["region"] = "ap-northeast-1";
+    params["use-transfer-acceleration"] = "true";
+    params["profile"] = "production";
+
+    S3BinaryCacheStoreConfig config("s3", "tokyo-cache", params);
+
+    EXPECT_EQ(
+        config.cacheUri,
+        (ParsedURL{
+            .scheme = "s3",
+            .authority = ParsedURL::Authority{.host = "tokyo-cache"},
+            .query =
+                (StringMap) {
+                    {"region", "ap-northeast-1"},
+                    {"use-transfer-acceleration", "true"},
+                    {"profile", "production"},
+                },
+        }));
+
+    EXPECT_EQ(config.region.get(), "ap-northeast-1");
+    EXPECT_EQ(config.use_transfer_acceleration.get(), true);
+    EXPECT_EQ(config.profile.get(), "production");
+}
+
+/**
+ * Test default value for transfer acceleration
+ */
+TEST(S3BinaryCacheStore, transferAccelerationDefaultValue)
+{
+    S3BinaryCacheStoreConfig config("s3", "test-bucket", {});
+
+    // Default should be false
+    EXPECT_EQ(config.use_transfer_acceleration.get(), false);
+}
+#endif
+
 } // namespace nix

src/libstore-tests/s3-url.cc

@@ -104,7 +104,38 @@ INSTANTIATE_TEST_SUITE_P(
                     },
             },
             "with_absolute_endpoint_uri",
-        }),
+        }
+#if NIX_WITH_AWS_AUTH
+        ,
+        ParsedS3URLTestCase{
+            "s3://my-bucket/key?use-transfer-acceleration=true",
+            {
+                .bucket = "my-bucket",
+                .key = {"key"},
+                .use_transfer_acceleration = true,
+            },
+            "with_transfer_acceleration",
+        },
+        ParsedS3URLTestCase{
+            "s3://my-bucket/key?use-transfer-acceleration=false",
+            {
+                .bucket = "my-bucket",
+                .key = {"key"},
+                .use_transfer_acceleration = false,
+            },
+            "with_transfer_acceleration_false",
+        },
+        ParsedS3URLTestCase{
+            "s3://my-bucket/key?use-transfer-acceleration=1",
+            {
+                .bucket = "my-bucket",
+                .key = {"key"},
+                .use_transfer_acceleration = true,
+            },
+            "with_transfer_acceleration_numeric",
+        }
+#endif
+        ),
     [](const ::testing::TestParamInfo<ParsedS3URLTestCase> & info) { return info.param.description; });
 
 // Parameterized test for invalid S3 URLs
@@ -138,6 +169,62 @@ INSTANTIATE_TEST_SUITE_P(
         InvalidS3URLTestCase{"s3://bucket", "error: URI has a missing or invalid key", "missing_key"}),
     [](const ::testing::TestParamInfo<InvalidS3URLTestCase> & info) { return info.param.description; });
 
+#if NIX_WITH_AWS_AUTH
+// =============================================================================
+// Transfer Acceleration Bucket Name Validation Tests
+// =============================================================================
+
+struct InvalidTransferAccelerationBucketTestCase
+{
+    std::string url;
+    std::string expectedErrorSubstring;
+    std::string description;
+};
+
+class InvalidTransferAccelerationBucketTest
+    : public ::testing::WithParamInterface<InvalidTransferAccelerationBucketTestCase>,
+      public ::testing::Test
+{};
+
+TEST_P(InvalidTransferAccelerationBucketTest, rejectsInvalidBucketNames)
+{
+    const auto & testCase = GetParam();
+    auto parsed = ParsedS3URL::parse(parseURL(testCase.url));
+
+    ASSERT_THAT(
+        [&parsed]() { parsed.toHttpsUrl(); },
+        ::testing::ThrowsMessage<Error>(testing::HasSubstrIgnoreANSIMatcher(testCase.expectedErrorSubstring)));
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    InvalidBucketNames,
+    InvalidTransferAccelerationBucketTest,
+    ::testing::Values(
+        InvalidTransferAccelerationBucketTestCase{
+            "s3://bucket.with.dots/key?use-transfer-acceleration=true",
+            "is not compatible with S3 Transfer Acceleration",
+            "bucket_with_dots",
+        },
+        InvalidTransferAccelerationBucketTestCase{
+            "s3://my.bucket.name/key?use-transfer-acceleration=true",
+            "is not compatible with S3 Transfer Acceleration",
+            "bucket_with_multiple_dots",
+        },
+        InvalidTransferAccelerationBucketTestCase{
+            "s3://bucket.name/key?use-transfer-acceleration=true",
+            "is not compatible with S3 Transfer Acceleration",
+            "bucket_with_single_dot",
+        },
+        InvalidTransferAccelerationBucketTestCase{
+            "s3://bucket/key?use-transfer-acceleration=true&endpoint=minio.local",
+            "cannot be used with custom endpoints",
+            "acceleration_with_custom_endpoint",
+        }),
+    [](const ::testing::TestParamInfo<InvalidTransferAccelerationBucketTestCase> & info) {
+        return info.param.description;
+    });
+#endif
+
 // =============================================================================
 // S3 URL to HTTPS Conversion Tests
 // =============================================================================
@@ -272,7 +359,53 @@ INSTANTIATE_TEST_SUITE_P(
             },
             "https://s3.eu-west-1.amazonaws.com/versioned-bucket/path/to/object?versionId=version456",
             "with_region_and_versionId",
-        }),
+        },
+#if NIX_WITH_AWS_AUTH
+        S3ToHttpsConversionTestCase{
+            ParsedS3URL{
+                .bucket = "my-cache",
+                .key = {"nix", "store", "abc123.nar.xz"},
+                .use_transfer_acceleration = true,
+            },
+            ParsedURL{
+                .scheme = "https",
+                .authority = ParsedURL::Authority{.host = "my-cache.s3-accelerate.amazonaws.com"},
+                .path = {"", "nix", "store", "abc123.nar.xz"},
+            },
+            "https://my-cache.s3-accelerate.amazonaws.com/nix/store/abc123.nar.xz",
+            "transfer_acceleration_enabled",
+        },
+        S3ToHttpsConversionTestCase{
+            ParsedS3URL{
+                .bucket = "tokyo-cache",
+                .key = {"key.txt"},
+                .region = "ap-northeast-1",
+                .use_transfer_acceleration = true,
+            },
+            ParsedURL{
+                .scheme = "https",
+                .authority = ParsedURL::Authority{.host = "tokyo-cache.s3-accelerate.amazonaws.com"},
+                .path = {"", "key.txt"},
+            },
+            "https://tokyo-cache.s3-accelerate.amazonaws.com/key.txt",
+            "transfer_acceleration_with_region",
+        },
+        S3ToHttpsConversionTestCase{
+            ParsedS3URL{
+                .bucket = "my-bucket",
+                .key = {"file"},
+                .use_transfer_acceleration = false,
+            },
+            ParsedURL{
+                .scheme = "https",
+                .authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
+                .path = {"", "my-bucket", "file"},
+            },
+            "https://s3.us-east-1.amazonaws.com/my-bucket/file",
+            "transfer_acceleration_explicitly_disabled",
+        }
+#endif
+        ),
     [](const ::testing::TestParamInfo<S3ToHttpsConversionTestCase> & info) { return info.param.description; });
 
 } // namespace nix

src/libstore/include/nix/store/s3-binary-cache-store.hh

@@ -113,11 +113,38 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
           https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html
         )"};
 
+#if NIX_WITH_AWS_AUTH
+    const Setting<bool> use_transfer_acceleration{
+        this,
+        false,
+        "use-transfer-acceleration",
+        R"(
+          Whether to use AWS S3 Transfer Acceleration for faster uploads and downloads
+          by routing transfers through CloudFront edge locations. This is particularly
+          useful when accessing S3 buckets from geographically distant locations.
+
+          When enabled, requests are routed to `bucket-name.s3-accelerate.amazonaws.com`
+          instead of the standard regional endpoint.
+
+          > **Note**
+          >
+          > Transfer Acceleration does not support bucket names with dots (periods).
+          > Bucket names like `my.bucket.name` will not work with this setting.
+          >
+          > This setting only applies to AWS S3 buckets. It has no effect when using
+          > custom endpoints for S3-compatible services.
+          >
+          > Additional charges apply for Transfer Acceleration. See AWS documentation
+          > for pricing details.
+        )"};
+#endif
+
     /**
      * Set of settings that are part of the S3 URI itself.
      * These are needed for region specification and other S3-specific settings.
      */
-    const std::set<const AbstractSetting *> s3UriSettings = {&profile, &region, &scheme, &endpoint};
+    const std::set<const AbstractSetting *> s3UriSettings = {
+        &profile, &region, &scheme, &endpoint, &use_transfer_acceleration};
 
     static const std::string name()
     {

src/libstore/include/nix/store/s3-url.hh

@@ -27,6 +27,9 @@ struct ParsedS3URL
     std::optional<std::string> region;
     std::optional<std::string> scheme;
     std::optional<std::string> versionId;
+#if NIX_WITH_AWS_AUTH
+    bool use_transfer_acceleration;
+#endif
     /**
      * The endpoint can be either missing, be an absolute URI (with a scheme like `http:`)
      * or an authority (so an IP address or a registered name).

src/libstore/s3-binary-cache-store.md

@@ -84,6 +84,27 @@ Your account will need an IAM policy to support uploading to the bucket:
 }
 ```
 
+### S3 Transfer Acceleration
+
+For faster uploads and downloads when accessing S3 buckets from geographically distant locations, you can enable AWS S3 Transfer Acceleration. This routes transfers through CloudFront edge locations for improved performance.
+
+To enable transfer acceleration, add the `use-transfer-acceleration=true` parameter to your S3 URL:
+
+```console
+$ nix copy nixpkgs.hello \
+  --to 's3://example-nix-cache?use-transfer-acceleration=true&region=ap-northeast-1'
+```
+
+#### Requirements for Transfer Acceleration
+
+- Bucket names cannot contain dots (`.`) - e.g., `my.bucket.name` will not work
+- Transfer Acceleration must be enabled on your S3 bucket (see [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/transfer-acceleration.html))
+- Additional charges apply for Transfer Acceleration (see AWS pricing)
+
+> **Note**
+>
+> Transfer Acceleration only works with AWS S3 buckets. It has no effect when using custom endpoints for S3-compatible services like MinIO.
+
 ### Examples
 
 With bucket policies and authentication set up as described above, uploading works via [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md) (experimental).
@@ -102,4 +123,11 @@ With bucket policies and authentication set up as described above, uploading wor
     's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com'
   ```
 
+- To use S3 Transfer Acceleration for faster transfers from distant locations:
+
+  ```console
+  $ nix copy nixpkgs.hello \
+    --to 's3://my-cache?profile=cache-upload&region=ap-northeast-1&use-transfer-acceleration=true'
+  ```
+
 )"