Skip to content

Commit d2c8ae9

Browse files
zoewanggzoewangg
authored
Merge pull request #3656 from aws/zoewang/fixSigV4aChecksum
Updated the SDK to calculate checksums for checksum required sigv4a operations
2 parents fe15787 + 8315cfc commit d2c8ae9

File tree

9 files changed

+428
-60
lines changed

9 files changed

+428
-60
lines changed

‎.changes/2.30.19.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,12 @@
4949
"category": "OpenSearch Service Serverless",
5050
"contributor": "",
5151
"description": "Custom OpenSearchServerless Entity ID for SAML Config."
52+
},
53+
{
54+
"type": "bugfix",
55+
"category": "Amazon S3",
56+
"contributor": "",
57+
"description": "Fixed an issue in the S3 client where it skipped checksum calculation for operations that use SigV4a signing and require checksums. See [#5878](https://github.com/aws/aws-sdk-java-v2/issues/5878)."
5258
}
5359
]
5460
}

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,10 @@
3232
- ### Features
3333
- Custom OpenSearchServerless Entity ID for SAML Config.
3434

35+
## __Amazon S3__
36+
- ### Bugfixes
37+
- Fixed an issue in the S3 client where it skipped checksum calculation for operations that use SigV4a signing and require checksums. See [#5878](https://github.com/aws/aws-sdk-java-v2/issues/5878).
38+
3539
# __2.30.18__ __2025-02-11__
3640
## __AWS AppSync__
3741
- ### Features

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/DefaultAwsCrtV4aHttpSigner.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import static software.amazon.awssdk.http.auth.aws.crt.internal.util.CrtHttpRequestConverter.toRequest;
2626
import static software.amazon.awssdk.http.auth.aws.crt.internal.util.CrtUtils.sanitizeRequest;
2727
import static software.amazon.awssdk.http.auth.aws.crt.internal.util.CrtUtils.toCredentials;
28+
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.ChecksumUtil.checksummer;
2829
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.ChecksumUtil.hasChecksumHeader;
2930
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.ChecksumUtil.isPayloadSigning;
3031
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.ChecksumUtil.useChunkEncoding;
@@ -44,6 +45,7 @@
4445
import software.amazon.awssdk.crt.http.HttpRequest;
4546
import software.amazon.awssdk.http.ContentStreamProvider;
4647
import software.amazon.awssdk.http.SdkHttpRequest;
48+
import software.amazon.awssdk.http.auth.aws.internal.signer.Checksummer;
4749
import software.amazon.awssdk.http.auth.aws.internal.signer.CredentialScope;
4850
import software.amazon.awssdk.http.auth.aws.signer.AwsV4aHttpSigner;
4951
import software.amazon.awssdk.http.auth.aws.signer.RegionSet;
@@ -68,10 +70,11 @@ public final class DefaultAwsCrtV4aHttpSigner implements AwsV4aHttpSigner {
6870

6971
@Override
7072
public SignedRequest sign(SignRequest<? extends AwsCredentialsIdentity> request) {
73+
Checksummer checksummer = checksummer(request, null);
7174
V4aProperties v4aProperties = v4aProperties(request);
7275
AwsSigningConfig signingConfig = signingConfig(request, v4aProperties);
7376
V4aPayloadSigner payloadSigner = v4aPayloadSigner(request, v4aProperties);
74-
return doSign(request, signingConfig, payloadSigner);
77+
return doSign(request, checksummer, signingConfig, payloadSigner);
7578
}
7679

7780
@Override
@@ -205,18 +208,21 @@ private static void configurePayloadSigning(AwsSigningConfig signingConfig, bool
205208
}
206209

207210
private static SignedRequest doSign(SignRequest<? extends AwsCredentialsIdentity> request,
211+
Checksummer checksummer,
208212
AwsSigningConfig signingConfig,
209213
V4aPayloadSigner payloadSigner) {
214+
215+
SdkHttpRequest.Builder requestBuilder = request.request().toBuilder();
210216
ContentStreamProvider contentStreamProvider = request.payload().orElse(null);
217+
211218
if (isAnonymous(request.identity())) {
212219
return SignedRequest.builder()
213220
.request(request.request())
214221
.payload(contentStreamProvider)
215222
.build();
216223
}
217224

218-
SdkHttpRequest.Builder requestBuilder = request.request().toBuilder();
219-
225+
checksummer.checksum(contentStreamProvider, requestBuilder);
220226
payloadSigner.beforeSigning(requestBuilder, contentStreamProvider, signingConfig.getSignedBodyValue());
221227

222228
SdkHttpRequest sdkHttpRequest = requestBuilder.build();

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java‎

Lines changed: 2 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,14 @@
1515

1616
package software.amazon.awssdk.http.auth.aws.internal.signer;
1717

18+
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.ChecksumUtil.checksummer;
1819
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.ChecksumUtil.hasChecksumHeader;
20+
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.ChecksumUtil.isEventStreaming;
1921
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.ChecksumUtil.isPayloadSigning;
2022
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.ChecksumUtil.useChunkEncoding;
2123
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.CredentialUtils.sanitizeCredentials;
2224
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.OptionalDependencyLoaderUtil.getEventStreamV4PayloadSigner;
2325
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.PRESIGN_URL_MAX_EXPIRATION_DURATION;
24-
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.STREAMING_EVENTS_PAYLOAD;
25-
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.STREAMING_SIGNED_PAYLOAD;
26-
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.STREAMING_SIGNED_PAYLOAD_TRAILER;
27-
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.STREAMING_UNSIGNED_PAYLOAD_TRAILER;
28-
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.UNSIGNED_PAYLOAD;
2926
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.X_AMZ_TRAILER;
3027

3128
import java.time.Clock;
@@ -35,7 +32,6 @@
3532
import java.util.function.Function;
3633
import software.amazon.awssdk.annotations.SdkInternalApi;
3734
import software.amazon.awssdk.http.ContentStreamProvider;
38-
import software.amazon.awssdk.http.Header;
3935
import software.amazon.awssdk.http.SdkHttpRequest;
4036
import software.amazon.awssdk.http.auth.aws.internal.signer.util.CredentialUtils;
4137
import software.amazon.awssdk.http.auth.aws.signer.AwsV4HttpSigner;
@@ -129,51 +125,6 @@ private static V4RequestSigner v4RequestSigner(
129125
return requestSigner.apply(v4Properties);
130126
}
131127

132-
private static Checksummer checksummer(BaseSignRequest<?, ? extends AwsCredentialsIdentity> request,
133-
Boolean isPayloadSigningOverride) {
134-
boolean isPayloadSigning = isPayloadSigningOverride != null ? isPayloadSigningOverride : isPayloadSigning(request);
135-
boolean isEventStreaming = isEventStreaming(request.request());
136-
boolean hasChecksumHeader = hasChecksumHeader(request);
137-
boolean isChunkEncoding = request.requireProperty(CHUNK_ENCODING_ENABLED, false);
138-
boolean isTrailing = request.request().firstMatchingHeader(X_AMZ_TRAILER).isPresent();
139-
boolean isFlexible = request.hasProperty(CHECKSUM_ALGORITHM) && !hasChecksumHeader;
140-
boolean isAnonymous = CredentialUtils.isAnonymous(request.identity());
141-
142-
if (isEventStreaming) {
143-
return Checksummer.forPrecomputed256Checksum(STREAMING_EVENTS_PAYLOAD);
144-
}
145-
146-
if (isPayloadSigning) {
147-
if (isChunkEncoding) {
148-
if (isFlexible || isTrailing) {
149-
return Checksummer.forPrecomputed256Checksum(STREAMING_SIGNED_PAYLOAD_TRAILER);
150-
}
151-
return Checksummer.forPrecomputed256Checksum(STREAMING_SIGNED_PAYLOAD);
152-
}
153-
154-
if (isFlexible) {
155-
return Checksummer.forFlexibleChecksum(request.property(CHECKSUM_ALGORITHM));
156-
}
157-
return Checksummer.create();
158-
}
159-
160-
if (isFlexible || isTrailing) {
161-
if (isChunkEncoding) {
162-
return Checksummer.forPrecomputed256Checksum(STREAMING_UNSIGNED_PAYLOAD_TRAILER);
163-
}
164-
}
165-
166-
if (isFlexible) {
167-
return Checksummer.forFlexibleChecksum(UNSIGNED_PAYLOAD, request.property(CHECKSUM_ALGORITHM));
168-
}
169-
170-
if (isAnonymous) {
171-
return Checksummer.forNoOp();
172-
}
173-
174-
return Checksummer.forPrecomputed256Checksum(UNSIGNED_PAYLOAD);
175-
}
176-
177128
/**
178129
* This is needed because of the pre-existing gap (pre-SRA) in behavior where we don't treat async + streaming + http +
179130
* unsigned-payload as signed-payload (fallback). We have to do some finagling of the payload-signing options before
@@ -310,8 +261,4 @@ private static Duration validateExpirationDuration(Duration expirationDuration)
310261
private static boolean isBetweenInclusive(Duration start, Duration x, Duration end) {
311262
return start.compareTo(x) <= 0 && x.compareTo(end) <= 0;
312263
}
313-
314-
private static boolean isEventStreaming(SdkHttpRequest request) {
315-
return "application/vnd.amazon.eventstream".equals(request.firstMatchingHeader(Header.CONTENT_TYPE).orElse(""));
316-
}
317264
}

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/ChecksumUtil.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,14 @@
1515

1616
package software.amazon.awssdk.http.auth.aws.internal.signer.util;
1717

18+
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.STREAMING_EVENTS_PAYLOAD;
19+
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.STREAMING_SIGNED_PAYLOAD;
20+
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.STREAMING_SIGNED_PAYLOAD_TRAILER;
21+
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.STREAMING_UNSIGNED_PAYLOAD_TRAILER;
22+
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.UNSIGNED_PAYLOAD;
23+
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.X_AMZ_TRAILER;
1824
import static software.amazon.awssdk.http.auth.aws.signer.AwsV4FamilyHttpSigner.CHECKSUM_ALGORITHM;
25+
import static software.amazon.awssdk.http.auth.aws.signer.AwsV4FamilyHttpSigner.CHUNK_ENCODING_ENABLED;
1926
import static software.amazon.awssdk.http.auth.aws.signer.AwsV4FamilyHttpSigner.PAYLOAD_SIGNING_ENABLED;
2027

2128
import java.io.InputStream;
@@ -24,6 +31,9 @@
2431
import software.amazon.awssdk.annotations.SdkInternalApi;
2532
import software.amazon.awssdk.checksums.SdkChecksum;
2633
import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm;
34+
import software.amazon.awssdk.http.Header;
35+
import software.amazon.awssdk.http.SdkHttpRequest;
36+
import software.amazon.awssdk.http.auth.aws.internal.signer.Checksummer;
2737
import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.ConstantChecksum;
2838
import software.amazon.awssdk.http.auth.spi.signer.BaseSignRequest;
2939
import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity;
@@ -141,4 +151,53 @@ public static boolean isPayloadSigning(BaseSignRequest<?, ? extends AwsCredentia
141151

142152
return isPayloadSigningEnabled;
143153
}
154+
155+
public static boolean isEventStreaming(SdkHttpRequest request) {
156+
return "application/vnd.amazon.eventstream".equals(request.firstMatchingHeader(Header.CONTENT_TYPE).orElse(""));
157+
}
158+
159+
public static Checksummer checksummer(BaseSignRequest<?, ? extends AwsCredentialsIdentity> request,
160+
Boolean isPayloadSigningOverride) {
161+
boolean isPayloadSigning = isPayloadSigningOverride != null ? isPayloadSigningOverride : isPayloadSigning(request);
162+
boolean isEventStreaming = isEventStreaming(request.request());
163+
boolean hasChecksumHeader = hasChecksumHeader(request);
164+
boolean isChunkEncoding = request.requireProperty(CHUNK_ENCODING_ENABLED, false);
165+
boolean isTrailing = request.request().firstMatchingHeader(X_AMZ_TRAILER).isPresent();
166+
boolean isFlexible = request.hasProperty(CHECKSUM_ALGORITHM) && !hasChecksumHeader;
167+
boolean isAnonymous = CredentialUtils.isAnonymous(request.identity());
168+
169+
if (isEventStreaming) {
170+
return Checksummer.forPrecomputed256Checksum(STREAMING_EVENTS_PAYLOAD);
171+
}
172+
173+
if (isPayloadSigning) {
174+
if (isChunkEncoding) {
175+
if (isFlexible || isTrailing) {
176+
return Checksummer.forPrecomputed256Checksum(STREAMING_SIGNED_PAYLOAD_TRAILER);
177+
}
178+
return Checksummer.forPrecomputed256Checksum(STREAMING_SIGNED_PAYLOAD);
179+
}
180+
181+
if (isFlexible) {
182+
return Checksummer.forFlexibleChecksum(request.property(CHECKSUM_ALGORITHM));
183+
}
184+
return Checksummer.create();
185+
}
186+
187+
if (isFlexible || isTrailing) {
188+
if (isChunkEncoding) {
189+
return Checksummer.forPrecomputed256Checksum(STREAMING_UNSIGNED_PAYLOAD_TRAILER);
190+
}
191+
}
192+
193+
if (isFlexible) {
194+
return Checksummer.forFlexibleChecksum(UNSIGNED_PAYLOAD, request.property(CHECKSUM_ALGORITHM));
195+
}
196+
197+
if (isAnonymous) {
198+
return Checksummer.forNoOp();
199+
}
200+
201+
return Checksummer.forPrecomputed256Checksum(UNSIGNED_PAYLOAD);
202+
}
144203
}

‎core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/crt/TestUtils.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public static <T extends AwsCredentialsIdentity> SignRequest<T> generateBasicReq
6060
.uri(URI.create("https://demo.us-east-1.amazonaws.com"))
6161
.build()
6262
.copy(requestOverrides))
63-
.payload(() -> new ByteArrayInputStream("{\"TableName\": \"foo\"}".getBytes()))
63+
.payload(() -> new ByteArrayInputStream("Hello world".getBytes()))
6464
.putProperty(REGION_SET, RegionSet.create("aws-global"))
6565
.putProperty(SERVICE_SIGNING_NAME, "demo")
6666
.putProperty(SIGNING_CLOCK, new TickingClock(Instant.ofEpochMilli(1596476903000L)))

‎core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/DefaultAwsCrtV4aHttpSignerTest.java‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@
1919
import static org.junit.jupiter.api.Assertions.assertNull;
2020
import static org.junit.jupiter.api.Assertions.assertThrows;
2121
import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32;
22+
import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32C;
23+
import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC64NVME;
24+
import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA1;
25+
import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA256;
2226
import static software.amazon.awssdk.crt.auth.signing.AwsSigningConfig.AwsSignatureType.HTTP_REQUEST_VIA_HEADERS;
2327
import static software.amazon.awssdk.crt.auth.signing.AwsSigningConfig.AwsSignatureType.HTTP_REQUEST_VIA_QUERY_PARAMS;
2428
import static software.amazon.awssdk.crt.auth.signing.AwsSigningConfig.AwsSignedBodyValue.STREAMING_AWS4_ECDSA_P256_SHA256_PAYLOAD;
@@ -45,8 +49,15 @@
4549
import java.time.Duration;
4650
import java.time.Instant;
4751
import java.util.List;
52+
import java.util.Locale;
4853
import java.util.Map;
54+
import java.util.stream.Stream;
55+
import org.assertj.core.api.AssertionsForClassTypes;
4956
import org.junit.jupiter.api.Test;
57+
import org.junit.jupiter.params.ParameterizedTest;
58+
import org.junit.jupiter.params.provider.Arguments;
59+
import org.junit.jupiter.params.provider.MethodSource;
60+
import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm;
5061
import software.amazon.awssdk.crt.auth.signing.AwsSigningConfig;
5162
import software.amazon.awssdk.http.Header;
5263
import software.amazon.awssdk.http.SdkHttpMethod;
@@ -57,6 +68,7 @@
5768
import software.amazon.awssdk.http.auth.spi.signer.SignRequest;
5869
import software.amazon.awssdk.http.auth.spi.signer.SignedRequest;
5970
import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity;
71+
import software.amazon.awssdk.utils.ImmutableMap;
6072

6173

6274
/**
@@ -66,6 +78,19 @@ public class DefaultAwsCrtV4aHttpSignerTest {
6678

6779
DefaultAwsCrtV4aHttpSigner signer = new DefaultAwsCrtV4aHttpSigner();
6880

81+
private static final Map<ChecksumAlgorithm, String> ALGORITHM_TO_VALUE = ImmutableMap.<ChecksumAlgorithm, String>builder()
82+
.put(CRC32, "i9aeUg==")
83+
.put(CRC32C, "crUfeA==")
84+
.put(SHA1, "e1AsOh9IyGCa4hLN+2Od7jlnP14=")
85+
.put(SHA256,
86+
"ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=")
87+
.put(CRC64NVME, "OOJZ0D8xKts=")
88+
.build();
89+
90+
public static Stream<Map.Entry<ChecksumAlgorithm, String>> checksumAlgorithmToValueParams() {
91+
return ALGORITHM_TO_VALUE.entrySet().stream();
92+
}
93+
6994
@Test
7095
public void sign_withBasicRequest_shouldSignWithHeaders() {
7196
AwsCredentialsIdentity credentials =
@@ -356,4 +381,35 @@ public void sign_WithPayloadSigningFalseAndChunkEncodingTrueWithoutTrailer_Deleg
356381
assertThat(signedRequest.request().firstMatchingHeader("x-amz-content-sha256")).hasValue("UNSIGNED-PAYLOAD");
357382
assertThat(signedRequest.request().firstMatchingHeader("x-amz-decoded-content-length")).isNotPresent();
358383
}
384+
385+
@ParameterizedTest
386+
@MethodSource("checksumAlgorithmToValueParams")
387+
public void sign_checksumAlgorithmPresent_shouldAddChecksumHeader(Map.Entry<ChecksumAlgorithm, String> checksumToValue) {
388+
ChecksumAlgorithm checksumAlgorithm = checksumToValue.getKey();
389+
SignRequest<? extends AwsCredentialsIdentity> request = generateBasicRequest(
390+
AwsCredentialsIdentity.create("access", "secret"),
391+
httpRequest -> {
392+
},
393+
signRequest -> signRequest.putProperty(CHECKSUM_ALGORITHM, checksumAlgorithm)
394+
);
395+
396+
SignedRequest signedRequest = signer.sign(request);
397+
assertThat(signedRequest.request().firstMatchingHeader("x-amz-checksum-" + checksumAlgorithm.algorithmId()
398+
.toLowerCase(Locale.US)))
399+
.contains(checksumToValue.getValue());
400+
}
401+
402+
@Test
403+
public void sign_checksumValueProvided_shouldNotOverrideChecksumHeader() {
404+
SignRequest<? extends AwsCredentialsIdentity> request = generateBasicRequest(
405+
AwsCredentialsIdentity.create("access", "secret"),
406+
httpRequest -> httpRequest
407+
.putHeader("x-amz-checksum-crc32", "some value"),
408+
signRequest -> signRequest.putProperty(CHECKSUM_ALGORITHM, CRC32)
409+
);
410+
411+
SignedRequest signedRequest = signer.sign(request);
412+
assertThat(signedRequest.request().firstMatchingHeader("x-amz-checksum-crc32"))
413+
.contains("some value");
414+
}
359415
}

0 commit comments

Comments
 (0)