[XEN-3146] Upgrade to AWS SDK 2.x

gradle.properties

@@ -5,7 +5,7 @@ junitJupiterVersion=5.4.2
 mockitoVersion=2.27.0
 solrVersion=6.6.5
 assVersion=2.0.8.2
-amazonVersion=1.12.782
+amazonVersion=2.35.7
 jaxBVersion=4.0.5
 restAssuredVersion=5.5.1
 awaitablityVersion=4.3.0

integration-tests/build.gradle

@@ -28,11 +28,15 @@ dependencies {
     testImplementation "org.junit.jupiter:junit-jupiter-params:${junitJupiterVersion}"
     testImplementation "org.awaitility:awaitility:${awaitablityVersion}"
 
-    testImplementation platform("com.amazonaws:aws-java-sdk-bom:${amazonVersion}")
+    testImplementation platform("software.amazon.awssdk:bom:${amazonVersion}")
 
-    testImplementation('com.amazonaws:aws-java-sdk-core')
-    testImplementation('com.amazonaws:aws-java-sdk-s3')
-    testImplementation("com.amazonaws:aws-java-sdk-sts")
+    testImplementation('software.amazon.awssdk:aws-core')
+    testImplementation('software.amazon.awssdk:s3')
+    testImplementation("software.amazon.awssdk:sts")
+
+    testImplementation "software.amazon.awssdk:apache-client:${amazonVersion}"
+
+    testImplementation "software.amazon.awssdk:netty-nio-client:${amazonVersion}"
 
     testRuntimeOnly "org.glassfish.jaxb:jaxb-runtime:${jaxBVersion}"
 }

integration-tests/src/test/java/eu/xenit/solr/backup/s3/SolrBackupTest.java

@@ -1,13 +1,5 @@
 package eu.xenit.solr.backup.s3;
 
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.auth.AWSStaticCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.client.builder.AwsClientBuilder;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3ClientBuilder;
-import com.amazonaws.services.s3.model.ObjectListing;
 import groovy.util.logging.Slf4j;
 import io.restassured.RestAssured;
 import io.restassured.builder.RequestSpecBuilder;
@@ -20,7 +12,20 @@
 import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestMethodOrder;
-
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.awscore.client.builder.AwsSyncClientBuilder;
+import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
+import software.amazon.awssdk.services.s3.S3Configuration;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.concurrent.TimeUnit;
 
 import static io.restassured.RestAssured.given;
@@ -37,11 +42,11 @@ class SolrBackupTest {
     static RequestSpecification specBackupDetails;
     static RequestSpecification specRestore;
     static RequestSpecification specRestoreStatus;
-    static AmazonS3 s3Client;
+    static S3Client s3Client;
     static final String BUCKET = "bucket";
 
     @BeforeEach
-    public void setup() {
+    public void setup() throws URISyntaxException {
         String basePathSolr = "solr/alfresco";
         String basePathSolrBackup = "solr/alfresco/replication";
         String solrHost = System.getProperty("solr.host", "localhost");
@@ -148,14 +153,22 @@ void testBackupWithNumberToLiveEndpoint() {
 
     void validateSnapshotCount(long count) {
         await().atMost(180, TimeUnit.SECONDS)
-                .until(() -> s3Client.listObjects(BUCKET)
-                        .getObjectSummaries()
+                /*
+                 * SDK v2 Migration:
+                 * - Switched to `ListObjectsV2Request` for the S3 call.
+                 * - The response object's method to get the list of objects is now `contents()`, not `objectSummaries()`.
+                 * - The object class is `S3Object`, which has the same `size()` and `key()` methods.
+                 */
+                .until(() -> s3Client.listObjectsV2(ListObjectsV2Request.builder().bucket(BUCKET)
+                                .build())
+                        .contents()
                         .stream()
-                        .filter(s3ObjectSummary -> s3ObjectSummary.getSize() == 0
-                                && s3ObjectSummary.getKey().contains("snapshot"))
+                        .filter(s3Object -> s3Object.size() == 0
+                                && s3Object.key().contains("snapshot"))
                         .count() == count);
 
     }
+
     private void callBackupEndpoint() {
         callBackupEndpoint(0);
     }
@@ -187,13 +200,30 @@ private void callBackupEndpoint(int count) {
                 });
     }
 
-    private AmazonS3 createInternalClient(
-            String region, String endpoint, String accessKey, String secretKey) {
-        ClientConfiguration clientConfig = new ClientConfiguration().withProtocol(Protocol.HTTPS);
-        AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard().withClientConfiguration(clientConfig);
-        clientBuilder.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)));
-        clientBuilder.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, region));
-        clientBuilder.withPathStyleAccessEnabled(true);
+    private S3Client createInternalClient(
+            String region, String endpoint, String accessKey, String secretKey) throws URISyntaxException {
+        // SDK v2 Migration: Removed explicit protocol setting, as it's inferred from the endpoint URI.
+        ClientOverrideConfiguration clientConfig = ClientOverrideConfiguration.builder().build();
+
+        S3Configuration configuration = S3Configuration.builder()
+                .build();
+
+        S3ClientBuilder clientBuilder = S3Client.builder()
+                .httpClientBuilder(ApacheHttpClient.builder()).overrideConfiguration(clientConfig)
+                .serviceConfiguration(configuration);
+        clientBuilder.credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secretKey)));
+
+        /*
+         * SDK v2 Migration:
+         * - Replaced the v1 `setEndpointConfiguration` with `endpointOverride` and `region`.
+         * - `endpointOverride` takes a URI object.
+         * - `region` must be set separately.
+         */
+        clientBuilder.endpointOverride(new URI(endpoint));
+        clientBuilder.region(Region.of(region));
+
+        // SDK v2 Migration: Replaced `pathStyleAccessEnabled(true)` with `forcePathStyle(true)`.
+        clientBuilder.forcePathStyle(true);
         return clientBuilder.build();
     }
 

integration-tests/src/test/resources/compose/docker-compose.yml

@@ -45,6 +45,7 @@ services:
       - S3_ACCESS_KEY=access_key
       - S3_SECRET_KEY=secret_key
       - S3_PATH_STYLE_ACCESS_ENABLED=true
+      - S3_CLIENT_CHECKSUM_VALIDATION_ENABLED=false
 
   localstack:
     container_name: localstack
@@ -57,6 +58,6 @@ services:
       - AWS_SECRET_ACCESS_KEY=secret_key
       - AWS_DEFAULT_REGION=us-east-1
     volumes:
-      - ./aws:/etc/localstack/init/ready.d
+      - ./aws:/etc/localstack/init/ready.d:r,Z
 
 

integration-tests/src/test/resources/solr.xml

@@ -10,7 +10,10 @@
             <str name="s3.secret.key">${S3_SECRET_KEY:}</str>
             <str name="s3.proxy.host">${S3_PROXY_HOST:}</str>
             <int name="s3.proxy.port">${S3_PROXY_PORT:0}</int>
+
             <bool name="s3.path.style.access.enabled">${S3_PATH_STYLE_ACCESS_ENABLED:false}</bool>
+            <bool name="s3.client.checksumValidationEnabled">${S3_CLIENT_CHECKSUM_VALIDATION_ENABLED:true}</bool>
+            <int name="s3.client.progressLogByteInterval">${S3_CLIENT_PROGRESS_LOG_BYTE_INTERVAL:4194304}</int>
         </repository>
     </backup>
 </solr>

solr-backup/build.gradle

@@ -20,11 +20,17 @@ dependencies {
     }
     compileOnly "org.alfresco:alfresco-search:${assVersion}"
 
-    implementation platform("com.amazonaws:aws-java-sdk-bom:${amazonVersion}")
+    implementation platform("software.amazon.awssdk:bom:${amazonVersion}")
 
-    implementation('com.amazonaws:aws-java-sdk-core')
-    implementation('com.amazonaws:aws-java-sdk-s3')
-    implementation("com.amazonaws:aws-java-sdk-sts")
+    implementation('ch.qos.logback:logback-classic:1.4.14')
+    compileOnly 'org.projectlombok:lombok:1.18.32'
+    annotationProcessor 'org.projectlombok:lombok:1.18.32'
+
+    implementation('software.amazon.awssdk:aws-core')
+    implementation('software.amazon.awssdk:s3')
+    implementation("software.amazon.awssdk:sts")
+    implementation "software.amazon.awssdk:apache-client:${amazonVersion}"
+    implementation "software.amazon.awssdk:netty-nio-client:${amazonVersion}"
     testImplementation("org.apache.solr:solr-core:${solrVersion}") {
         exclude group: 'org.restlet.jee' // Only available in JCenter, not essential in this project.
     }

solr-backup/src/main/java/eu/xenit/solr/backup/s3/ProgressTrackingInputStream.java

@@ -0,0 +1,52 @@
+package eu.xenit.solr.backup.s3;
+
+import lombok.NonNull;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.function.Consumer;
+
+/**
+ * Custom InputStream wrapper that reports the number of bytes read to a listener.
+ */
+public class ProgressTrackingInputStream extends FilterInputStream {
+
+    private final @NonNull Consumer<Long> listener;
+    private long bytesRead = 0;
+
+    /**
+     * Creates a {@code FilterInputStream}
+     * by assigning the  argument {@code in}
+     * to the field {@code this.in} so as
+     * to remember it for later use.
+     *
+     * @param in the underlying input stream, or {@code null} if
+     *           this instance is to be created without an underlying stream.
+     * @param listener the listener to report the number of bytes read to
+     */
+    protected ProgressTrackingInputStream(InputStream in, @NonNull Consumer<Long> listener) {
+        super(in);
+        this.listener = listener;
+    }
+
+    @Override
+    public int read() throws IOException {
+        int b = super.read();
+        if (b != -1) {
+            bytesRead += 1;
+            listener.accept(bytesRead);
+        }
+        return b;
+    }
+
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException {
+        int bytes = super.read(b, off, len);
+        if (bytes > 0) {
+            bytesRead += bytes;
+            listener.accept(bytesRead);
+        }
+        return bytes;
+    }
+}

solr-backup/src/main/java/eu/xenit/solr/backup/s3/S3BackupRepository.java

@@ -66,12 +66,16 @@ public void init(NamedList args) {
     this.config = args;
     S3BackupRepositoryConfig backupConfig = new S3BackupRepositoryConfig(this.config);
 
-    // If a client was already created, close it to avoid any resource leak
+      // If a client was already created, close it to avoid any resource leak
     if (client != null) {
       client.close();
     }
 
-    this.client = backupConfig.buildClient();
+      try {
+          this.client = backupConfig.buildClient();
+      } catch (URISyntaxException e) {
+          throw new RuntimeException(e);
+      }
   }
 
   @Override

solr-backup/src/main/java/eu/xenit/solr/backup/s3/S3BackupRepositoryConfig.java

@@ -16,31 +16,48 @@
  */
 package eu.xenit.solr.backup.s3;
 
+import lombok.Getter;
 import org.apache.solr.common.util.NamedList;
 
+import java.net.URISyntaxException;
+
 /**
  * Class representing the {@code backup} S3 config bundle specified in solr.xml. All user-provided
  * config can be overridden via environment variables (use uppercase, with '_' instead of '.'), see
  * {@link S3BackupRepositoryConfig#toEnvVar}.
  */
+@Getter
 public class S3BackupRepositoryConfig {
     public static final String S3_BUCKET_NAME = "s3.bucket.name";
     public static final String S3_REGION = "s3.region";
     public static final String S3_ACCESS_KEY = "s3.access.key";
     public static final String S3_SECRET_KEY = "s3.secret.key";
     public static final String S3_ENDPOINT = "s3.endpoint";
+    public static final String S3_PATH_STYLE_ACCESS_ENABLED = "s3.path.style.access.enabled";
+    public static final String S3_CLIENT_CHECKSUM_VALIDATION_ENABLED = "s3.client.checksum.validation.enabled";
     public static final String S3_PROXY_HOST = "s3.proxy.host";
     public static final String S3_PROXY_PORT = "s3.proxy.port";
-    public static final String S3_PATH_STYLE_ACCESS_ENABLED = "s3.path.style.access.enabled";
+    public static final String S3_CLIENT_PROGRESS_LOG_BYTE_INTERVAL = "s3.client.progressLogByteInterval";
 
     private final String bucketName;
+
     private final String region;
+
     private final String accessKey;
+
     private final String secretKey;
+
     private final String proxyHost;
+
     private final int proxyPort;
+
     private final String endpoint;
-    private final Boolean pathStyleAccessEnabled;
+
+    private final boolean pathStyleAccessEnabled;
+
+    private final boolean checksumValidationEnabled;
+
+    private final int progressLogByteInterval;
 
 
     public S3BackupRepositoryConfig(NamedList<?> config) {
@@ -52,13 +69,15 @@ public S3BackupRepositoryConfig(NamedList<?> config) {
         accessKey = getStringConfig(config, S3_ACCESS_KEY);
         secretKey = getStringConfig(config, S3_SECRET_KEY);
         pathStyleAccessEnabled = getBooleanConfig(config, S3_PATH_STYLE_ACCESS_ENABLED);
+        checksumValidationEnabled = getBooleanConfig(config, S3_CLIENT_CHECKSUM_VALIDATION_ENABLED);
+        progressLogByteInterval = getIntConfig(config, S3_CLIENT_PROGRESS_LOG_BYTE_INTERVAL);
     }
 
     /**
      * @return a {@link S3StorageClient} from the provided config.
      */
-    public S3StorageClient buildClient() {
-        return new S3StorageClient(bucketName, region, proxyHost, proxyPort, endpoint, accessKey, secretKey, pathStyleAccessEnabled);
+    public S3StorageClient buildClient() throws URISyntaxException {
+        return new S3StorageClient(this);
     }
 
     private static String getStringConfig(NamedList<?> config, String property) {