Skip to content

Commit 95eddaa

Browse files
GarrettBeattyGarrettBeatty
committed
Add multipartdownload with openstreamwithresponse api
update to basedownloadrequest
1 parent 42e25ce commit 95eddaa

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+8267
-96
lines changed

‎generator/.DevConfigs/9d07dc1e-d82d-4f94-8700-c7b57f872042.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"services": [
3+
{
4+
"serviceName": "S3",
5+
"type": "minor",
6+
"changeLogMessages": [
7+
"Create OpenStreamWithResponseAsync API and implement multi part download"
8+
]
9+
}
10+
]
11+
}

‎generator/ServiceClientGeneratorLib/Generators/SourceFiles/AssemblyInfo.cs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -94,9 +94,10 @@ public override string TransformText()
9494
#line hidden
9595
this.Write(@"[assembly: InternalsVisibleTo(""AWSSDK.UnitTests.S3.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4"")]
9696
[assembly: InternalsVisibleTo(""AWSSDK.UnitTests.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4"")]
97+
[assembly: InternalsVisibleTo(""DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7"")]
9798
");
9899

99-
#line 32 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
100+
#line 33 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
100101
}
101102

102103
#line default
@@ -125,14 +126,14 @@ public override string TransformText()
125126
// [assembly: AssemblyVersion(""1.0.*"")]
126127
[assembly: AssemblyVersion(""");
127128

128-
#line 55 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
129+
#line 56 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
129130
this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.ServiceVersion));
130131

131132
#line default
132133
#line hidden
133134
this.Write("\")]\r\n[assembly: AssemblyFileVersion(\"");
134135

135-
#line 56 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
136+
#line 57 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
136137
this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.ServiceFileVersion));
137138

138139
#line default

‎generator/ServiceClientGeneratorLib/Generators/SourceFiles/AssemblyInfo.tt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ using System.Runtime.CompilerServices;
2929
<# if (this.Config.AssemblyTitle=="AWSSDK.S3") { #>
3030
[assembly: InternalsVisibleTo("AWSSDK.UnitTests.S3.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4")]
3131
[assembly: InternalsVisibleTo("AWSSDK.UnitTests.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4")]
32+
[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")]
3233
<# } #>
3334
[assembly: AssemblyConfiguration("")]
3435
[assembly: AssemblyProduct("Amazon Web Services SDK for .NET")]

‎sdk/src/Services/S3/Custom/Transfer/Internal/BaseCommand.cs‎

Lines changed: 4 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -38,43 +38,11 @@ internal abstract partial class BaseCommand<TResponse> where TResponse : class
3838
{
3939
internal GetObjectRequest ConvertToGetObjectRequest(BaseDownloadRequest request)
4040
{
41-
GetObjectRequest getRequest = new GetObjectRequest()
42-
{
43-
BucketName = request.BucketName,
44-
Key = request.Key,
45-
VersionId = request.VersionId
46-
};
47-
((Amazon.Runtime.Internal.IAmazonWebServiceRequest)getRequest).AddBeforeRequestHandler(this.RequestEventHandler);
48-
49-
if (request.IsSetModifiedSinceDate())
50-
{
51-
getRequest.ModifiedSinceDate = request.ModifiedSinceDate;
52-
}
53-
if (request.IsSetUnmodifiedSinceDate())
54-
{
55-
getRequest.UnmodifiedSinceDate = request.UnmodifiedSinceDate;
56-
}
57-
58-
getRequest.ServerSideEncryptionCustomerMethod = request.ServerSideEncryptionCustomerMethod;
59-
getRequest.ServerSideEncryptionCustomerProvidedKey = request.ServerSideEncryptionCustomerProvidedKey;
60-
getRequest.ServerSideEncryptionCustomerProvidedKeyMD5 = request.ServerSideEncryptionCustomerProvidedKeyMD5;
61-
getRequest.ChecksumMode = request.ChecksumMode;
62-
getRequest.RequestPayer = request.RequestPayer;
63-
64-
if (request.IsSetExpectedBucketOwner())
65-
{
66-
getRequest.ExpectedBucketOwner = request.ExpectedBucketOwner;
67-
}
68-
if (request.IsSetIfMatch())
69-
{
70-
getRequest.EtagToMatch = request.IfMatch;
71-
}
72-
if (request.IsSetIfNoneMatch())
73-
{
74-
getRequest.EtagToNotMatch = request.IfNoneMatch;
75-
}
41+
// Use centralized request mapping
42+
GetObjectRequest getRequest = RequestMapper.MapToGetObjectRequest(request);
7643

77-
getRequest.ResponseHeaderOverrides = request.ResponseHeaderOverrides;
44+
// Add command-specific event handler
45+
((Amazon.Runtime.Internal.IAmazonWebServiceRequest)getRequest).AddBeforeRequestHandler(this.RequestEventHandler);
7846

7947
return getRequest;
8048
}

‎sdk/src/Services/S3/Custom/Transfer/Internal/BufferedDataSource.cs‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
/*******************************************************************************
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
4+
* this file except in compliance with the License. A copy of the License is located at
5+
*
6+
* http://aws.amazon.com/apache2.0
7+
*
8+
* or in the "license" file accompanying this file.
9+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
10+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
* specific language governing permissions and limitations under the License.
12+
* *****************************************************************************
13+
* __ _ _ ___
14+
* ( )( \/\/ )/ __)
15+
* /__\ \ / \__ \
16+
* (_)(_) \/\/ (___/
17+
*
18+
* AWS SDK for .NET
19+
* API Version: 2006-03-01
20+
*
21+
*/
22+
using System;
23+
using System.Diagnostics.CodeAnalysis;
24+
using System.Threading;
25+
using System.Threading.Tasks;
26+
27+
namespace Amazon.S3.Transfer.Internal
28+
{
29+
/// <summary>
30+
/// ArrayPool-based buffered data source that reads from pre-buffered part data.
31+
/// Manages ArrayPool lifecycle and provides efficient buffer-to-buffer copying.
32+
/// </summary>
33+
internal class BufferedDataSource : IPartDataSource
34+
{
35+
private readonly StreamPartBuffer _partBuffer;
36+
private bool _disposed = false;
37+
38+
public int PartNumber => _partBuffer.PartNumber;
39+
public bool IsComplete => _partBuffer.RemainingBytes == 0;
40+
41+
public BufferedDataSource(StreamPartBuffer partBuffer)
42+
{
43+
_partBuffer = partBuffer ?? throw new ArgumentNullException(nameof(partBuffer));
44+
}
45+
46+
public Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
47+
{
48+
ThrowIfDisposed();
49+
50+
try
51+
{
52+
if (buffer == null)
53+
throw new ArgumentNullException(nameof(buffer));
54+
if (offset < 0)
55+
throw new ArgumentOutOfRangeException(nameof(offset), "Offset must be non-negative");
56+
if (count < 0)
57+
throw new ArgumentOutOfRangeException(nameof(count), "Count must be non-negative");
58+
if (offset + count > buffer.Length)
59+
throw new ArgumentException("Offset and count exceed buffer bounds");
60+
61+
if (_partBuffer.RemainingBytes == 0)
62+
{
63+
return Task.FromResult(0); // End of part
64+
}
65+
66+
// Calculate bytes to copy from buffered part
67+
var availableBytes = _partBuffer.RemainingBytes;
68+
var bytesToRead = Math.Min(count, availableBytes);
69+
70+
Buffer.BlockCopy(
71+
_partBuffer.ArrayPoolBuffer, // Source: ArrayPool buffer
72+
_partBuffer.CurrentPosition, // Source offset
73+
buffer, // Destination: user buffer
74+
offset, // Destination offset
75+
bytesToRead // Bytes to copy
76+
);
77+
78+
// Update position in the part buffer
79+
_partBuffer.CurrentPosition += bytesToRead;
80+
81+
return Task.FromResult(bytesToRead);
82+
}
83+
catch (Exception)
84+
{
85+
// On any error during read (including validation), mark the buffer as consumed to prevent further reads
86+
_partBuffer.CurrentPosition = _partBuffer.Length;
87+
throw;
88+
}
89+
}
90+
91+
private void ThrowIfDisposed()
92+
{
93+
if (_disposed)
94+
throw new ObjectDisposedException(nameof(BufferedDataSource));
95+
}
96+
97+
[SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Dispose methods should not throw exceptions")]
98+
public void Dispose()
99+
{
100+
if (!_disposed)
101+
{
102+
try
103+
{
104+
// Dispose the underlying StreamPartBuffer, which returns ArrayPool buffer to pool
105+
_partBuffer?.Dispose();
106+
}
107+
catch (Exception)
108+
{
109+
// Suppressing CA1031: Dispose methods should not throw exceptions
110+
// Continue disposal process silently on any errors
111+
}
112+
113+
_disposed = true;
114+
}
115+
}
116+
}
117+
}

‎sdk/src/Services/S3/Custom/Transfer/Internal/BufferedDownloadConfiguration.cs‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/*******************************************************************************
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
4+
* this file except in compliance with the License. A copy of the License is located at
5+
*
6+
* http://aws.amazon.com/apache2.0
7+
*
8+
* or in the "license" file accompanying this file.
9+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
10+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
* specific language governing permissions and limitations under the License.
12+
* *****************************************************************************
13+
* __ _ _ ___
14+
* ( )( \/\/ )/ __)
15+
* /__\ \ / \__ \
16+
* (_)(_) \/\/ (___/
17+
*
18+
* AWS SDK for .NET
19+
* API Version: 2006-03-01
20+
*
21+
*/
22+
using System;
23+
24+
namespace Amazon.S3.Transfer.Internal
25+
{
26+
/// <summary>
27+
/// Configuration settings for buffered multipart downloads to streams.
28+
/// Extends base coordinator settings with buffer-specific parameters.
29+
/// </summary>
30+
internal class BufferedDownloadConfiguration : DownloadCoordinatorConfiguration
31+
{
32+
/// <summary>
33+
/// Maximum parts to keep in memory simultaneously.
34+
/// </summary>
35+
public int MaxInMemoryParts { get; set; }
36+
37+
/// <summary>
38+
/// Buffer size for I/O operations.
39+
/// </summary>
40+
public int BufferSize { get; set; }
41+
42+
/// <summary>
43+
/// Creates a BufferedDownloadConfiguration with the specified configuration values.
44+
/// </summary>
45+
/// <param name="concurrentServiceRequests">Maximum concurrent HTTP requests for downloading parts.</param>
46+
/// <param name="maxInMemoryParts">Maximum number of parts to keep in memory simultaneously.</param>
47+
/// <param name="bufferSize">Buffer size used for optimal I/O operations.</param>
48+
/// <param name="targetPartSizeBytes">Target size for each part in bytes.</param>
49+
/// <exception cref="ArgumentOutOfRangeException">Thrown when any parameter is less than or equal to 0.</exception>
50+
public BufferedDownloadConfiguration(
51+
int concurrentServiceRequests,
52+
int maxInMemoryParts,
53+
int bufferSize,
54+
long targetPartSizeBytes)
55+
: base(concurrentServiceRequests, targetPartSizeBytes)
56+
{
57+
if (maxInMemoryParts <= 0)
58+
throw new ArgumentOutOfRangeException(nameof(maxInMemoryParts), "Must be greater than 0");
59+
if (bufferSize <= 0)
60+
throw new ArgumentOutOfRangeException(nameof(bufferSize), "Must be greater than 0");
61+
62+
MaxInMemoryParts = maxInMemoryParts;
63+
BufferSize = bufferSize;
64+
}
65+
}
66+
}

0 commit comments

Comments
 (0)