Merge pull request #1070 from SUSE/handle-nested-debian

[6/x] Mirroring nested debian structure

lib/rmt/cli/export.rb

@@ -34,7 +34,7 @@ def repos(path)
     begin
       suma_product_tree.mirror
     rescue RMT::Mirror::Exception => e
-      logger.warn(e.message)
+      logger.warn(_('Exporting SUSE Manager product tree failed: %{error_message}') % { error_message: e.message })
     end
 
     repos_file = File.join(path, 'repos.json')

lib/rmt/cli/import.rb

@@ -25,7 +25,7 @@ def repos(path)
           url: suma_repo_url
         ).mirror
       rescue RMT::Mirror::Exception => e
-        logger.warn(e.message)
+        logger.warn(_('Importing suma product tree failed: %{error_message}') % { error_message: e.message })
       end
 
       repos = JSON.parse(File.read(repos_file))

lib/rmt/mirror.rb

@@ -37,6 +37,9 @@ def repository_mirror_class
   end
 
   def repository_type
+    # We search repomd structure first since it is more common
+    # Debian is less common
+
     search = {
       repomd: File.join(repository.external_url, RPM_FILE_NEEDLE),
       debian: File.join(repository.external_url, DEB_FILE_NEEDLE)

lib/rmt/mirror/base.rb

@@ -80,7 +80,6 @@ def repository_path(*args)
     File.join(mirroring_base_dir, repository.local_path, *args)
   end
 
-  # FIXME: Write some specs for me!
   def create_repository_path
     FileUtils.mkpath(repository_path) unless Dir.exist?(repository_path)
   rescue StandardError => e
@@ -111,8 +110,8 @@ def cleanup_temp_dirs
     @temp_dirs = {}
   end
 
-  def enqueue(ref)
-    @enqueued << ref
+  def enqueue(refs)
+    @enqueued.concat(Array(refs))
   end
 
   def download_enqueued(continue_on_error: false)
@@ -155,7 +154,7 @@ def replace_directory(source:, destination:, with_backup: true, &block)
 
   def copy_directory_content(source:, destination:)
     replace_directory(source: source, destination: destination, with_backup: false) do
-      FileUtils.mv(Dir.glob(source), destination)
+      FileUtils.mv(Dir.glob(source), destination, force: true)
     end
   end
 end

lib/rmt/mirror/debian.rb

@@ -3,6 +3,8 @@ class RMT::Mirror::Debian < RMT::Mirror::Base
   SIGNATURE_FILE_NAME = 'Release.gpg'.freeze
   KEY_FILE_NAME = 'Release.key'.freeze
   INRELEASE_FILE_NAME = 'InRelease'.freeze
+  NESTED_REPOSITORY_REGEX = %r{/dists/.*/$}.freeze
+  DETECT_NONMANDATORY_FILES = %r{/(Packages|Sources|Translation)(-\w+)?$/}.freeze
 
   def mirror_implementation
     create_repository_path
@@ -28,9 +30,17 @@ def mirror_metadata
     # We need to make sure downloading the InRelease file which is not referenced
     # anywhere
     metadata_refs << inrelease
-    metadata_refs.each { |ref| enqueue(ref) }
 
+    # The nested debian structure only contains the zipped version of packages sometimes
+    # However, the release file still contains a reference to the unzipped versions
+    # So, we don't error if they don't exist
+    packages, remaining = metadata_refs.partition { |ref| ref.relative_path.match(DETECT_NONMANDATORY_FILES) }
+    enqueue(packages)
+    download_enqueued(continue_on_error: true)
+
+    enqueue(remaining)
     download_enqueued
+
     metadata_refs
   end
 
@@ -60,6 +70,17 @@ def parse_package_list(packagelist)
         ref.size = current[:size].to_i
         ref.type = :deb
 
+        # In a nested debian repository stucture, the metadata and packages are stored in different locations
+        # so we need to update the base_url if we encounter the nested structure
+        # We assume that if the base_url contains '/dists/', it's a nested debian structure
+        if ref.base_url.match?(NESTED_REPOSITORY_REGEX)
+          ref.tap do |r|
+            r.base_url.sub!(NESTED_REPOSITORY_REGEX, '/')
+            r.base_dir.sub!(NESTED_REPOSITORY_REGEX, '/')
+            r.cache_dir.sub!(NESTED_REPOSITORY_REGEX, '/')
+          end
+        end
+
         packages << ref
         current = {}
       end

lib/rmt/mirror/license.rb

@@ -18,6 +18,7 @@ def licenses_available?
     end
     request.run
 
+    logger.debug("No license directory found for repository '#{uri}'")
     false
   end
 

spec/lib/rmt/cli/export_spec.rb

@@ -95,7 +95,7 @@
         allow_any_instance_of(RMT::Mirror).to receive(:mirror_now)
 
         expect(suma_product_tree_double).to receive(:mirror).and_raise(RMT::Mirror::Exception, 'black mirror')
-        expect_any_instance_of(RMT::Logger).to receive(:warn).with('black mirror')
+        expect_any_instance_of(RMT::Logger).to receive(:warn).with(/black mirror/)
         command
       end
     end

spec/lib/rmt/file_validator_spec.rb

@@ -238,30 +238,6 @@ def initialize(deep_verify)
     end
   end
 
-  RSpec::Matchers.define :contain_records_like do |expected|
-    match do |actual|
-      record_struct = Struct.new(:local_path, :checksum, :checksum_type, :size)
-
-      @actual = actual.map { |r| record_struct.new(r.local_path, r.checksum, r.checksum_type, r.size) }
-      @expected = expected.map { |r| record_struct.new(r.local_path, r.checksum, r.checksum_type, r.size) }
-
-      actual.all? do |record|
-        expected.any? do |object|
-          record.local_path == object.local_path &&
-            record.checksum == object.checksum &&
-            record.checksum_type == object.checksum_type &&
-            record.size == object.size
-        end
-      end
-    end
-
-    failure_message do |actual|
-      "expected that collection #{actual} would contain #{expected}"
-    end
-
-    diffable
-  end
-
   describe '#find_valid_files_by_checksum' do
     let(:valid_file) do
       fixture_path = file_fixture('dummy_product/product/apples-0.1-0.x86_64.rpm').to_s

spec/lib/rmt/mirror/base_spec.rb

@@ -1,9 +1,9 @@
 require 'rails_helper'
 
 describe RMT::Mirror::Base do
-  subject(:base) { described_class.new(**configuration) }
+  subject(:base) { described_class.new(**mirror_configuration) }
 
-  let(:configuration) do
+  let(:mirror_configuration) do
     {
       repository: repository,
       logger: logger,
@@ -86,6 +86,21 @@
     end
   end
 
+  describe '#create_repository_path' do
+    it 'creates the repository path' do
+      allow(Dir).to receive(:exist?).and_return(false)
+      expect(FileUtils).to receive(:mkpath).with('/rspec/repository/update/hype/15.3/product/')
+      base.create_repository_path
+    end
+
+    it 'fails when it could not create the repository path' do
+      allow(Dir).to receive(:exist?).and_return(false)
+      allow(FileUtils).to receive(:mkpath).and_raise(StandardError)
+
+      expect { base.create_repository_path }.to raise_error(RMT::Mirror::Exception, /Could not create local directory/)
+    end
+  end
+
   describe '#cleanup_temp_dirs' do
     let(:temp_metadata) { '/tmp/metadata' }
     let(:temp_licenses) { '/tmp/licenses' }
@@ -124,15 +139,15 @@
   end
 
   describe '#check_signature' do
-    let(:config) do
+    let(:ref_configuration) do
       {
         base_dir: '/tmp',
         base_url: 'https://updates.suse.de/'
       }
     end
-    let(:signature_file) { RMT::Mirror::FileReference.new(relative_path: 'repo.gpg', **config) }
-    let(:key_file) { RMT::Mirror::FileReference.new(relative_path: 'repo.key', **config) }
-    let(:metadata) { RMT::Mirror::FileReference.new(relative_path: 'metadata', **config) }
+    let(:signature_file) { RMT::Mirror::FileReference.new(relative_path: 'repo.gpg', **ref_configuration) }
+    let(:key_file) { RMT::Mirror::FileReference.new(relative_path: 'repo.key', **ref_configuration) }
+    let(:metadata) { RMT::Mirror::FileReference.new(relative_path: 'metadata', **ref_configuration) }
     let(:gpg_checker) do
       RMT::GPG.new(
         metadata_file: metadata.local_path,
@@ -177,6 +192,7 @@
 
     it 'downloads enqueued contents and clear queue' do
       expect(downloader).to receive(:download_multi)
+      expect(base.enqueued.count).to eq(1)
       base.download_enqueued
       expect(base.enqueued).to be_empty
     end
@@ -236,7 +252,7 @@
 
     it 'copies content from source to destination without backup' do
       expect(base).to receive(:replace_directory).with(source: src, destination: dest, with_backup: false).and_yield
-      expect(FileUtils).to receive(:mv).with(Dir.glob(src), dest)
+      expect(FileUtils).to receive(:mv).with(Dir.glob(src), dest, force: true)
       base.copy_directory_content(source: src, destination: dest)
     end
   end

spec/lib/rmt/mirror/debian_spec.rb

@@ -1,7 +1,7 @@
 require 'rails_helper'
 
 describe RMT::Mirror::Debian do
-  subject(:debian) { described_class.new(**configuration) }
+  subject(:debian) { described_class.new(**debian_mirror_configuration) }
 
   let(:repository) do
     create :repository,
@@ -11,7 +11,7 @@
 
   # Configuration for Debian mirroring instance
   let(:mirroring_base_dir) { '/test/repository/base/path/' }
-  let(:configuration) do
+  let(:debian_mirror_configuration) do
     {
       repository: repository,
       logger: RMT::Logger.new('/dev/null'),
@@ -83,14 +83,15 @@
   end
 
   describe '#mirror_metadata' do
-    let(:config) do
+    let(:release_fixture) { 'Release' }
+    let(:release_configuration) do
       {
-        relative_path: 'Release',
+        relative_path: release_fixture,
         base_dir: file_fixture('debian/'),
         base_url: 'https://updates.suse.de/Debian/'
       }
     end
-    let(:release_ref) { RMT::Mirror::FileReference.new(**config) }
+    let(:release_ref) { RMT::Mirror::FileReference.new(**release_configuration) }
 
     before do
       allow(debian).to receive(:temp).with(:metadata).and_return('bar')
@@ -100,7 +101,7 @@
     it 'succeeds' do
       allow(debian).to receive(:check_signature)
       allow(debian).to receive(:parse_release_file).and_return([])
-      expect(debian).to receive(:download_enqueued)
+      expect(debian).to receive(:download_enqueued).twice
       debian.mirror_metadata
     end
 
@@ -110,7 +111,36 @@
       it 'downloads and parses the file' do
         expect(debian).to receive(:download_cached!).with(release_path, to: 'bar')
         expect(debian).to receive(:check_signature)
+        expect(debian).to receive(:download_enqueued).twice
+        debian.mirror_metadata
+      end
+    end
+
+    context 'nested debian repository' do
+      let(:release_fixture) { 'nested/Release' }
+
+      before do
+        allow(debian).to receive(:download_cached!).and_return(release_ref)
+        allow(debian).to receive(:check_signature)
+        described_class.send(:public, :enqueued)
+      end
+
+      it 'ignores non-existent references from release file' do
+        allow(debian).to receive(:download_enqueued)
+        allow(debian).to receive(:enqueue)
+
+        expect(debian).to receive(:download_enqueued).with(continue_on_error: true)
+        expect(debian).to receive(:enqueue).with(all(be_like_relative_path(/(Packages|Sources|Translation)(-\w+)?$/)))
+
+        debian.mirror_metadata
+      end
+
+      it 'calls download_enqueued for the remaining valid paths' do
+        expect(debian).to receive(:download_enqueued).with(continue_on_error: true)
         expect(debian).to receive(:download_enqueued)
+        expect(debian).to receive(:enqueue).with(all(be_like_relative_path(/(Packages|Sources|Translation)(-\w+)?$/))).once
+        expect(debian).to receive(:enqueue).once
+
         debian.mirror_metadata
       end
     end
@@ -161,17 +191,38 @@
         expect { debian.parse_package_list(packages_ref) }.to raise_error(RMT::Mirror::Exception, /unexpected end of file/)
       end
     end
+
+    context 'nested repository structure' do
+      let(:fixture) { 'nested/Packages.gz' }
+      let(:repository) do
+        create :repository,
+          name: 'HYPE product repository debian 15.3',
+          external_url: 'https://ppa.launchpadcontent.net/ondrej/nginx/ubuntu/dists/focal/'
+      end
+
+      it 'removes dists/ from mirroring path and external URL' do
+        packages = debian.parse_package_list(packages_ref)
+
+        expect(packages).to all(
+          have_attributes(
+            base_url: 'https://ppa.launchpadcontent.net/ondrej/nginx/ubuntu/',
+            base_dir: mirroring_base_dir + 'ondrej/nginx/ubuntu/',
+            cache_dir: mirroring_base_dir + 'ondrej/nginx/ubuntu/'
+          )
+        )
+      end
+    end
   end
 
   describe '#parse_release_file' do
-    let(:config) do
+    let(:release_configuration) do
       {
         relative_path: rel_path,
         base_dir: file_fixture('debian/'),
         base_url: 'https://updates.suse.de/Debian/'
       }
     end
-    let(:release_ref) { RMT::Mirror::FileReference.new(**config) }
+    let(:release_ref) { RMT::Mirror::FileReference.new(**release_configuration) }
 
     context 'Release file is valid' do
       let(:rel_path) { 'Release' }

spec/lib/rmt/mirror/license_spec.rb

@@ -14,20 +14,22 @@
   let(:license_configuration) do
     {
       repository: repository,
-      logger: RMT::Logger.new('/dev/null'),
+      logger: logger,
       mirroring_base_dir: base_dir
     }
   end
 
+  let(:logger) { RMT::Logger.new('/dev/null') }
+
   let(:fixture) { 'directory.yast' }
-  let(:config) do
+  let(:license_listing_configuration) do
     {
       relative_path: fixture,
       base_dir: file_fixture(''),
       base_url: 'https://updates.suse.de/sles/'
     }
   end
-  let(:licenses_ref) { RMT::Mirror::FileReference.new(**config) }
+  let(:licenses_ref) { RMT::Mirror::FileReference.new(**license_listing_configuration) }
 
   before do
     allow(FileUtils).to receive(:mkpath).with(license.repository_path).and_return(nil)
@@ -42,6 +44,7 @@
 
     it 'returns false if directory.yast is not available' do
       stub_request(:head, license.repository_url('directory.yast')).to_return(status: 404, body: '', headers: {})
+      expect(logger).to receive(:debug)
       expect(license.licenses_available?).to eq(false)
     end