From 68d05c713143de6674fe5900a27a8349cfb076fb Mon Sep 17 00:00:00 2001 From: Felix Moehler Date: Fri, 28 Nov 2025 11:15:20 +0100 Subject: [PATCH 1/2] retry network interface deletion in case vm creation errors out --- .../lib/cloud/aws/network_interface.rb | 26 ++++++++++++--- .../spec/unit/network_interface_spec.rb | 32 +++++++++++++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/bosh_aws_cpi/lib/cloud/aws/network_interface.rb b/src/bosh_aws_cpi/lib/cloud/aws/network_interface.rb index d8c88d11..b07504d5 100644 --- a/src/bosh_aws_cpi/lib/cloud/aws/network_interface.rb +++ b/src/bosh_aws_cpi/lib/cloud/aws/network_interface.rb @@ -3,7 +3,7 @@ class NetworkInterface include Helpers CREATE_NETWORK_INTERFACE_WAIT_TIME = 30 - DELETE_NETWORK_INTERFACE_WAIT_TIME = 5 + DELETE_NETWORK_INTERFACE_WAIT_TIME = 10 def initialize(aws_network_interface, ec2_client, logger) @aws_network_interface = aws_network_interface @@ -60,10 +60,20 @@ def add_associate_public_ip_address(vm_type) end def delete - @logger.info("Deleting network_interface: #{@aws_network_interface.id}") - @aws_network_interface.delete - rescue Aws::EC2::Errors::InvalidNetworkInterfaceIDNotFound, Aws::EC2::Errors::InvalidParameterValue => e - @logger.warn("Network interface '#{@aws_network_interface.id}' could not be deleted: #{e.message}") + begin + @logger.info("Deleting network_interface: #{@aws_network_interface.id}") + errors = [Aws::EC2::Errors::InvalidNetworkInterfaceInUse, Aws::EC2::Errors::InvalidParameterValue] + + Bosh::Common.retryable(sleep: delete_network_interface_wait_time, tries: 50, on: errors) do |_tries, error| + if error.class == Aws::EC2::Errors::InvalidNetworkInterfaceInUse || error.class == Aws::EC2::Errors::InvalidParameterValue + @logger.warn("Network Interface was in use: #{error}. Retrying deletion after #{Bosh::AwsCloud::NetworkInterface::DELETE_NETWORK_INTERFACE_WAIT_TIME} seconds...") + end + @aws_network_interface.delete + true + end + rescue => e + @logger.warn("Failed to delete network interface '#{@aws_network_interface.id}' could not be deleted: #{e.inspect}") + end end def mac_address @@ -87,5 +97,11 @@ def nic_configuration(device_index) nic end + + private + + def delete_network_interface_wait_time + Bosh::AwsCloud::NetworkInterface::DELETE_NETWORK_INTERFACE_WAIT_TIME + end end end diff --git a/src/bosh_aws_cpi/spec/unit/network_interface_spec.rb b/src/bosh_aws_cpi/spec/unit/network_interface_spec.rb index 1c0b7839..da1bbf00 100644 --- a/src/bosh_aws_cpi/spec/unit/network_interface_spec.rb +++ b/src/bosh_aws_cpi/spec/unit/network_interface_spec.rb @@ -110,6 +110,38 @@ network_interface.delete expect(aws_network_interface).to have_received(:delete) end + + it 'retries on Aws::EC2::Errors::InvalidNetworkInterfaceInUse error' do + allow(network_interface).to receive(:delete_network_interface_wait_time).and_return(0) + + return_values = [:raise, true] + + expect(aws_network_interface).to receive(:delete).exactly(2).times do + return_value = return_values.shift + return_value == :raise ? raise(Aws::EC2::Errors::InvalidNetworkInterfaceInUse.new(nil, 'IP address is already in use')) : return_value + end + + network_interface.delete + end + + it 'retries on Aws::EC2::Errors::InvalidParameterValue error' do + allow(network_interface).to receive(:delete_network_interface_wait_time).and_return(0) + + return_values = [:raise, true] + + expect(aws_network_interface).to receive(:delete).exactly(2).times do + return_value = return_values.shift + return_value == :raise ? raise(Aws::EC2::Errors::InvalidParameterValue.new(nil, 'Network Interface is currently in use')) : return_value + end + + network_interface.delete + end + + it 'fails gracefully if deletion fails' do + allow(aws_network_interface).to receive(:delete).and_raise(StandardError.new('Deletion failed')) + network_interface.delete + expect(logger).to have_received(:warn).with(/Failed to delete network interface 'eni-12345' could not be deleted: #/) + end end describe '#mac_address' do From 8ddf3cb897e81e91039f275b86249136b3760b04 Mon Sep 17 00:00:00 2001 From: Felix Moehler Date: Tue, 2 Dec 2025 09:21:23 +0100 Subject: [PATCH 2/2] implement review --- src/bosh_aws_cpi/lib/cloud/aws/network_interface.rb | 12 +++--------- src/bosh_aws_cpi/spec/unit/network_interface_spec.rb | 4 ++-- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/bosh_aws_cpi/lib/cloud/aws/network_interface.rb b/src/bosh_aws_cpi/lib/cloud/aws/network_interface.rb index b07504d5..c575aa10 100644 --- a/src/bosh_aws_cpi/lib/cloud/aws/network_interface.rb +++ b/src/bosh_aws_cpi/lib/cloud/aws/network_interface.rb @@ -4,6 +4,7 @@ class NetworkInterface CREATE_NETWORK_INTERFACE_WAIT_TIME = 30 DELETE_NETWORK_INTERFACE_WAIT_TIME = 10 + RETRYABLE_ERRORS = [Aws::EC2::Errors::InvalidNetworkInterfaceInUse, Aws::EC2::Errors::InvalidParameterValue] def initialize(aws_network_interface, ec2_client, logger) @aws_network_interface = aws_network_interface @@ -62,10 +63,9 @@ def add_associate_public_ip_address(vm_type) def delete begin @logger.info("Deleting network_interface: #{@aws_network_interface.id}") - errors = [Aws::EC2::Errors::InvalidNetworkInterfaceInUse, Aws::EC2::Errors::InvalidParameterValue] - Bosh::Common.retryable(sleep: delete_network_interface_wait_time, tries: 50, on: errors) do |_tries, error| - if error.class == Aws::EC2::Errors::InvalidNetworkInterfaceInUse || error.class == Aws::EC2::Errors::InvalidParameterValue + Bosh::Common.retryable(sleep: Bosh::AwsCloud::NetworkInterface::DELETE_NETWORK_INTERFACE_WAIT_TIME, tries: 50, on: RETRYABLE_ERRORS) do |_tries, error| + if RETRYABLE_ERRORS.include?(error.class) @logger.warn("Network Interface was in use: #{error}. Retrying deletion after #{Bosh::AwsCloud::NetworkInterface::DELETE_NETWORK_INTERFACE_WAIT_TIME} seconds...") end @aws_network_interface.delete @@ -97,11 +97,5 @@ def nic_configuration(device_index) nic end - - private - - def delete_network_interface_wait_time - Bosh::AwsCloud::NetworkInterface::DELETE_NETWORK_INTERFACE_WAIT_TIME - end end end diff --git a/src/bosh_aws_cpi/spec/unit/network_interface_spec.rb b/src/bosh_aws_cpi/spec/unit/network_interface_spec.rb index da1bbf00..02cfbab4 100644 --- a/src/bosh_aws_cpi/spec/unit/network_interface_spec.rb +++ b/src/bosh_aws_cpi/spec/unit/network_interface_spec.rb @@ -112,7 +112,7 @@ end it 'retries on Aws::EC2::Errors::InvalidNetworkInterfaceInUse error' do - allow(network_interface).to receive(:delete_network_interface_wait_time).and_return(0) + stub_const('Bosh::AwsCloud::NetworkInterface::DELETE_NETWORK_INTERFACE_WAIT_TIME', 0) return_values = [:raise, true] @@ -125,7 +125,7 @@ end it 'retries on Aws::EC2::Errors::InvalidParameterValue error' do - allow(network_interface).to receive(:delete_network_interface_wait_time).and_return(0) + stub_const('Bosh::AwsCloud::NetworkInterface::DELETE_NETWORK_INTERFACE_WAIT_TIME', 0) return_values = [:raise, true]