moved classes into different packages. upgraded SDK. tested table create request factory

Author: Alexander Patrikalakis

README.md

@@ -17,21 +17,31 @@ The CLI's usage follows with required parameters marked by asterisks.
       Use this flag to use strongly consistent scan. If the flag is not used 
       it will default to eventually consistent scan
       Default: false
-    --createDestination
-      Create destination table if it does not exist
-      Default: false
     --copyStreamSpecificationWhenCreating
       Use the source table stream specification for the destination table 
       during its creation.
       Default: false
+    --createAllGsi
+      Create all GSI in destination table
+      Default: false
+    --createAllLsi
+      Create all LSI in destination table
+      Default: false
+    --createDestination
+      Create destination table if it does not exist
+      Default: false
     --destinationEndpoint
       Endpoint of the destination table
-  * --destinationRegion
+  * --destinationSigningRegion
       Signing region for the destination endpoint
   * --destinationTable
       Name of the destination table
     --help
       Display usage information
+    --includeGsi
+      Include the following GSI in the destination table
+    --includeLsi
+      Include the following LSI in the destination table
     --maxWriteThreads
       Number of max threads to write to destination table
       Default: 1024
@@ -44,7 +54,7 @@ The CLI's usage follows with required parameters marked by asterisks.
       Default: 0
     --sourceEndpoint
       Endpoint of the source table
-  * --sourceRegion
+  * --sourceSigningRegion
       Signing region for the source endpoint
   * --sourceTable
       Name of the source table
@@ -87,8 +97,8 @@ To transfer to a different region, create two AmazonDynamoDBClients
 with different endpoints to pass into the DynamoDBBootstrapWorker and the DynamoDBConsumer.
 
 ```java
-import com.amazonaws.dynamodb.bootstrap.DynamoDBBootstrapWorker;
-import com.amazonaws.dynamodb.bootstrap.DynamoDBConsumer;
+import DynamoDBBootstrapWorker;
+import com.amazonaws.dynamodb.bootstrap.consumer.DynamoDBConsumer;
 import com.amazonaws.dynamodb.bootstrap.exception.NullReadCapacityException;
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
@@ -98,28 +108,22 @@ import java.util.concurrent.Executors;
 
 class TransferDataFromOneTableToAnother {
     public static void main(String[] args) {
-        AmazonDynamoDB client = AmazonDynamoDBClientBuilder.standard()
+        final AmazonDynamoDB client = AmazonDynamoDBClientBuilder.standard()
                 .withRegion(com.amazonaws.regions.Regions.US_WEST_1).build();
-        DynamoDBBootstrapWorker worker = null;
         try {
             // 100.0 read operations per second. 4 threads to scan the table.
-            worker = new DynamoDBBootstrapWorker(client,
+            final DynamoDBBootstrapWorker worker = new DynamoDBBootstrapWorker(client,
                     100.0, "mySourceTable", 4);
+            // 50.0 write operations per second. 8 threads to scan the table.
+            final DynamoDBConsumer consumer = new DynamoDBConsumer(client, "myDestinationTable",
+                    50.0, Executors.newFixedThreadPool(8));
+            worker.pipe(consumer);
         } catch (NullReadCapacityException e) {
             System.err.println("The DynamoDB source table returned a null read capacity.");
             System.exit(1);
-        }
-        // 50.0 write operations per second. 8 threads to scan the table.
-        DynamoDBConsumer consumer = new DynamoDBConsumer(client, "myDestinationTable", 50.0,
-                Executors.newFixedThreadPool(8));
-        try {
-            worker.pipe(consumer);
-        } catch (ExecutionException e) {
+        } catch (ExecutionException | InterruptedException e) {
             System.err.println("Encountered exception when executing transfer: " + e.getMessage());
             System.exit(1);
-        } catch (InterruptedException e){
-            System.err.println("Interrupted when executing transfer: " + e.getMessage());
-            System.exit(1);
         }
     }
 }
@@ -128,13 +132,15 @@ class TransferDataFromOneTableToAnother {
 
 ### 2. Transfer Data From one DynamoDB Table to a Blocking Queue.
 
-The below example will read from a DynamoDB table and export to an array blocking queue. This is useful for when another application would like to consume
-the DynamoDB entries but does not have a setup application for it. They can just retrieve the queue (consumer.getQueue()) and then continually pop() from it
+The below example will read from a DynamoDB table and export to an array blocking queue.
+This is useful for when another application would like to consume
+the DynamoDB entries but does not have a setup application for it.
+They can just retrieve the queue (consumer.getQueue()) and then continually pop() from it
 to then process the new entries.
 
 ```java
-import com.amazonaws.dynamodb.bootstrap.BlockingQueueConsumer;
-import com.amazonaws.dynamodb.bootstrap.DynamoDBBootstrapWorker;
+import com.amazonaws.dynamodb.bootstrap.consumer.BlockingQueueConsumer;
+import DynamoDBBootstrapWorker;
 import com.amazonaws.dynamodb.bootstrap.exception.NullReadCapacityException;
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
@@ -143,29 +149,20 @@ import java.util.concurrent.ExecutionException;
 
 class TransferDataFromOneTableToBlockingQueue {
     public static void main(String[] args) {
-        AmazonDynamoDB client = AmazonDynamoDBClientBuilder.standard()
+        final AmazonDynamoDB client = AmazonDynamoDBClientBuilder.standard()
                 .withRegion(com.amazonaws.regions.Regions.US_WEST_1).build();
-
-        DynamoDBBootstrapWorker worker = null;
-
         try {
             // 100.0 read operations per second. 4 threads to scan the table.
-            worker = new DynamoDBBootstrapWorker(client, 100.0, "mySourceTable", 4);
+            final DynamoDBBootstrapWorker worker = new DynamoDBBootstrapWorker(client, 100.0,
+                    "mySourceTable", 4);
+            final BlockingQueueConsumer consumer = new BlockingQueueConsumer(8);
+            worker.pipe(consumer);
         } catch (NullReadCapacityException e) {
             System.err.println("The DynamoDB source table returned a null read capacity.");
             System.exit(1);
-        }
-
-        BlockingQueueConsumer consumer = new BlockingQueueConsumer(8);
-
-        try {
-            worker.pipe(consumer);
-        } catch (ExecutionException e) {
+        } catch (ExecutionException | InterruptedException e) {
             System.err.println("Encountered exception when executing transfer: " + e.getMessage());
             System.exit(1);
-        } catch (InterruptedException e){
-            System.err.println("Interrupted when executing transfer: " + e.getMessage());
-            System.exit(1);
         }
     }
 }

pom.xml

@@ -11,7 +11,7 @@
         <url>https://github.com/awslabs/dynamodb-import-export-tool.git</url>
     </scm>
     <properties>
-        <jdk.version>1.7</jdk.version>
+        <jdk.version>1.8</jdk.version>
         <aws.java.sdk.version>1.11.123</aws.java.sdk.version>
         <powermock.version>1.6.6</powermock.version>
         <jcommander.version>1.69</jcommander.version>

src/main/java/com/amazonaws/dynamodb/bootstrap/AbstractLogProvider.java

@@ -14,14 +14,14 @@
  */
 package com.amazonaws.dynamodb.bootstrap;
 
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-
+import com.amazonaws.dynamodb.bootstrap.constants.BootstrapConstants;
+import com.amazonaws.dynamodb.bootstrap.consumer.AbstractLogConsumer;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
 
-import com.amazonaws.dynamodb.bootstrap.constants.BootstrapConstants;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
 
 /**
  * Abstract class to send inputs from a source to a consumer.

src/main/java/com/amazonaws/dynamodb/bootstrap/CommandLineArgs.java

@@ -18,6 +18,8 @@
 import com.beust.jcommander.Parameter;
 import lombok.Getter;
 
+import java.util.List;
+
 /**
  * This class contains the parameters to input when executing the program from
  * command line.
@@ -60,6 +62,22 @@ public boolean getHelp() {
     @Parameter(names = CREATE_DESTINATION_TABLE_IF_MISSING, description = "Create destination table if it does not exist")
     private boolean createDestinationTableIfMissing;
 
+    public static final String CREATE_ALL_LSI = "--createAllLsi";
+    @Parameter(names = CREATE_ALL_LSI, description = "Create all LSI in destination table")
+    private boolean createAllLsi;
+
+    public static final String CREATE_ALL_GSI = "--createAllGsi";
+    @Parameter(names = CREATE_ALL_GSI, description = "Create all GSI in destination table")
+    private boolean createAllGsi;
+
+    public static final String INCLUDE_LSI = "--includeLsi";
+    @Parameter(names = INCLUDE_LSI, description = "Include the following LSI in the destination table")
+    private List<String> includeLsi;
+
+    public static final String INCLUDE_GSI = "--includeGsi";
+    @Parameter(names = INCLUDE_GSI, description = "Include the following GSI in the destination table")
+    private List<String> includeGsi;
+
     public static final String COPY_STREAM_SPECIFICATION_WHEN_CREATING = "--copyStreamSpecificationWhenCreating";
     @Parameter(names = COPY_STREAM_SPECIFICATION_WHEN_CREATING, description = "Use the source table stream specification for the destination table during its creation.")
     private boolean copyStreamSpecification;