retry network interface deletion in case vm creation errors out

src/bosh_aws_cpi/lib/cloud/aws/network_interface.rb

@@ -3,7 +3,8 @@ class NetworkInterface
     include Helpers
 
     CREATE_NETWORK_INTERFACE_WAIT_TIME = 30
-    DELETE_NETWORK_INTERFACE_WAIT_TIME = 5
+    DELETE_NETWORK_INTERFACE_WAIT_TIME = 10
+    RETRYABLE_ERRORS = [Aws::EC2::Errors::InvalidNetworkInterfaceInUse, Aws::EC2::Errors::InvalidParameterValue]
 
     def initialize(aws_network_interface, ec2_client, logger)
       @aws_network_interface = aws_network_interface
@@ -60,10 +61,19 @@ def add_associate_public_ip_address(vm_type)
     end
 
     def delete
-      @logger.info("Deleting network_interface: #{@aws_network_interface.id}")
-      @aws_network_interface.delete
-    rescue Aws::EC2::Errors::InvalidNetworkInterfaceIDNotFound, Aws::EC2::Errors::InvalidParameterValue => e
-      @logger.warn("Network interface '#{@aws_network_interface.id}' could not be deleted: #{e.message}")
+      begin
+        @logger.info("Deleting network_interface: #{@aws_network_interface.id}")
+
+        Bosh::Common.retryable(sleep: Bosh::AwsCloud::NetworkInterface::DELETE_NETWORK_INTERFACE_WAIT_TIME, tries: 50, on: RETRYABLE_ERRORS) do |_tries, error|
+          if RETRYABLE_ERRORS.include?(error.class)
+            @logger.warn("Network Interface was in use: #{error}. Retrying deletion after #{Bosh::AwsCloud::NetworkInterface::DELETE_NETWORK_INTERFACE_WAIT_TIME} seconds...")
+          end
+          @aws_network_interface.delete
+          true
+        end
+      rescue => e
+        @logger.warn("Failed to delete network interface '#{@aws_network_interface.id}' could not be deleted: #{e.inspect}")
+      end
     end
 
     def mac_address

src/bosh_aws_cpi/spec/unit/network_interface_spec.rb

@@ -110,6 +110,38 @@
       network_interface.delete
       expect(aws_network_interface).to have_received(:delete)
     end
+
+    it 'retries on Aws::EC2::Errors::InvalidNetworkInterfaceInUse error' do
+      stub_const('Bosh::AwsCloud::NetworkInterface::DELETE_NETWORK_INTERFACE_WAIT_TIME', 0)
+
+      return_values = [:raise, true]
+
+      expect(aws_network_interface).to receive(:delete).exactly(2).times do
+        return_value = return_values.shift
+        return_value == :raise ? raise(Aws::EC2::Errors::InvalidNetworkInterfaceInUse.new(nil, 'IP address is already in use')) : return_value
+      end
+
+      network_interface.delete
+    end
+
+    it 'retries on Aws::EC2::Errors::InvalidParameterValue error' do
+      stub_const('Bosh::AwsCloud::NetworkInterface::DELETE_NETWORK_INTERFACE_WAIT_TIME', 0)
+
+      return_values = [:raise, true]
+
+      expect(aws_network_interface).to receive(:delete).exactly(2).times do
+        return_value = return_values.shift
+        return_value == :raise ? raise(Aws::EC2::Errors::InvalidParameterValue.new(nil, 'Network Interface is currently in use')) : return_value
+      end
+
+      network_interface.delete
+    end
+
+    it 'fails gracefully if deletion fails' do
+      allow(aws_network_interface).to receive(:delete).and_raise(StandardError.new('Deletion failed'))
+      network_interface.delete
+      expect(logger).to have_received(:warn).with(/Failed to delete network interface 'eni-12345' could not be deleted: #<StandardError: Deletion failed>/)
+    end
   end
 
   describe '#mac_address' do