Merge pull request #1060 from SUSE/refactor_repomd_mirroring

[4/x] Refactor repomd mirroring

lib/rmt/mirror/base.rb

@@ -21,6 +21,9 @@ def initialize(repository:, logger:, mirroring_base_dir: RMT::DEFAULT_MIRROR_DIR
   end
 
   def mirror
+    # FIXME: stub me in specs!
+    create_repository_path
+    logger.info _('Mirroring repository %{repo} to %{dir}') % { repo: repository.name || repository_url, dir: repository_path }
     mirror_implementation
   rescue RMT::Mirror::Exception => e
     raise RMT::Mirror::Exception.new(_('Error while mirroring repository: %{error}' % { error: e.message }))
@@ -79,6 +82,15 @@ def repository_path(*args)
     File.join(mirroring_base_dir, repository.local_path, *args)
   end
 
+  # FIXME: Write some specs for me!
+  def create_repository_path
+    FileUtils.mkpath(repository_path) unless Dir.exist?(repository_path)
+  rescue StandardError => e
+    raise RMT::Mirror::Exception.new(
+      _('Could not create local directory %{dir} with error: %{error}') % { dir: repository_path, error: e.message }
+    )
+  end
+
   def create_temp_dir(name)
     temp_dirs[name] = Dir.mktmpdir(name.to_s)
   rescue StandardError => e

lib/rmt/mirror/license.rb

@@ -0,0 +1,44 @@
+class RMT::Mirror::License < RMT::Mirror::Base
+  DIRECTORY_YAST = 'directory.yast'.freeze
+  def repository_url(*args)
+    URI.join(repository.external_url.chomp('/') + '.license/', *args).to_s
+  end
+
+  def repository_path(*args)
+    File.join(mirroring_base_dir, repository.local_path.chomp('/') + '.license/', *args)
+  end
+
+  def licenses_available?
+    uri = URI.join(repository_url(DIRECTORY_YAST))
+    uri.query = repository.auth_token if repository.auth_token
+
+    request = RMT::HttpRequest.new(uri, method: :head, followlocation: true)
+    request.on_success do
+      return true
+    end
+    request.run
+
+    false
+  end
+
+  def mirror_implementation
+    return unless licenses_available?
+
+    create_temp_dir(:license)
+    directory_yast = download_cached!(DIRECTORY_YAST, to: temp(:license))
+
+    File.readlines(directory_yast.local_path)
+        .map(&:strip).reject { |item| item == 'directory.yast' }
+        .map { |relative_path| file_reference(relative_path, to: temp(:license)) }
+        .each { |ref| enqueue(ref) }
+
+    download_enqueued
+
+    replace_directory(source: temp(:license), destination: repository_path)
+  rescue RMT::Downloader::Exception => e
+    raise RMT::Mirror::Exception.new(_('Error while mirroring license files: %{error}') % { error: e.message })
+  end
+
+
+
+end

lib/rmt/mirror/repomd.rb

@@ -3,137 +3,57 @@
 require 'repomd_parser'
 require 'time'
 
-class RMT::Mirror::Repomd
-  include RMT::Deduplicator
-  include RMT::FileValidator
-
-  def initialize(logger:, mirroring_base_dir: RMT::DEFAULT_MIRROR_DIR, mirror_src: false, airgap_mode: false)
-    @mirroring_base_dir = mirroring_base_dir
-    @logger = logger
-    @mirror_src = mirror_src
-    @airgap_mode = airgap_mode
-    @deep_verify = false
-
-    # don't save files for deduplication when in offline mode
-    @downloader = RMT::Downloader.new(logger: logger, track_files: !airgap_mode)
-  end
-
-  def mirror(repository_url:, local_path:, auth_token: nil, repo_name: nil)
-    repository_dir = File.join(mirroring_base_dir, local_path)
+class RMT::Mirror::Repomd < RMT::Mirror::Base
 
-    logger.info _('Mirroring repository %{repo} to %{dir}') % { repo: repo_name || repository_url, dir: repository_dir }
+  def mirror_implementation
+    create_temp_dir(:metadata)
+    licenses = RMT::Mirror::License.new(repository: repository, logger: logger, mirroring_base_dir: mirroring_base_dir)
+    licenses.mirror
 
-    create_repository_dir(repository_dir)
-    temp_licenses_dir = create_temp_dir
-    # downloading license doesn't require an auth token
-    mirror_license(repository_dir, repository_url, temp_licenses_dir)
+    metadata_files = mirror_metadata
+    mirror_packages(metadata_files)
 
-    downloader.auth_token = auth_token
-    temp_metadata_dir = create_temp_dir
-    metadata_files = mirror_metadata(repository_dir, repository_url, temp_metadata_dir)
-    mirror_packages(metadata_files, repository_dir, repository_url)
-
-    replace_directory(temp_licenses_dir, repository_dir.chomp('/') + '.license/') if Dir.exist?(temp_licenses_dir)
-    replace_directory(File.join(temp_metadata_dir, 'repodata'), File.join(repository_dir, 'repodata'))
-  ensure
-    [temp_licenses_dir, temp_metadata_dir].each { |dir| FileUtils.remove_entry(dir, true) }
+    replace_directory(source: File.join(temp(:metadata), 'repodata'), destination: repository_path('repodata'))
   end
 
   protected
 
-  attr_reader :airgap_mode, :deep_verify, :downloader, :logger, :mirroring_base_dir, :mirror_src
-
-  def create_repository_dir(repository_dir)
-    FileUtils.mkpath(repository_dir) unless Dir.exist?(repository_dir)
-  rescue StandardError => e
-    raise RMT::Mirror::Exception.new(
-      _('Could not create local directory %{dir} with error: %{error}') % { dir: repository_dir, error: e.message }
-    )
-  end
-
-  def create_temp_dir
-    Dir.mktmpdir
-  rescue StandardError => e
-    raise RMT::Mirror::Exception.new(_('Could not create a temporary directory: %{error}') % { error: e.message })
-  end
-
-  def mirror_metadata(repository_dir, repository_url, temp_metadata_dir)
-    mirroring_paths = {
-      base_url: URI.join(repository_url),
-      base_dir: temp_metadata_dir,
-      cache_dir: repository_dir
-    }
-
-    repomd_xml = RMT::Mirror::FileReference.new(relative_path: 'repodata/repomd.xml', **mirroring_paths)
-    downloader.download_multi([repomd_xml])
-
-    begin
-      signature_file = RMT::Mirror::FileReference.new(relative_path: 'repodata/repomd.xml.asc', **mirroring_paths)
-      key_file       = RMT::Mirror::FileReference.new(relative_path: 'repodata/repomd.xml.key', **mirroring_paths)
-      # mirror repomd.xml.asc first, because there are repos with repomd.xml.asc but without repomd.xml.key
-      downloader.download_multi([signature_file])
-      downloader.download_multi([key_file])
-
-      RMT::GPG.new(
-        metadata_file: repomd_xml.local_path,
-        key_file: key_file.local_path,
-        signature_file: signature_file.local_path,
-        logger: logger
-      ).verify_signature
-    rescue RMT::Downloader::Exception => e
-      if (e.http_code == 404)
-        logger.info(_('Repository metadata signatures are missing'))
-      else
-        raise(_('Downloading repo signature/key failed with: %{message}, HTTP code %{http_code}') % { message: e.message, http_code: e.http_code })
-      end
-    end
+  def mirror_metadata
+    repomd_xml = download_cached!('repodata/repomd.xml', to: temp(:metadata))
+    signature_file = file_reference('repodata/repomd.xml.asc', to: temp(:metadata))
+    key_file = file_reference('repodata/repomd.xml.key', to: temp(:metadata))
+    check_signature(key_file: key_file, signature_file: signature_file, metadata_file: repomd_xml)
 
     metadata_files = RepomdParser::RepomdXmlParser.new(repomd_xml.local_path).parse
-      .map { |reference| RMT::Mirror::FileReference.build_from_metadata(reference, **mirroring_paths) }
+      .map do |reference|
+        ref = RMT::Mirror::FileReference.build_from_metadata(reference, base_dir: temp(:metadata), base_url: repomd_xml.base_url)
+        enqueue ref
+        ref
+      end
 
-    downloader.download_multi(metadata_files.dup)
+    download_enqueued
 
     metadata_files
   rescue StandardError => e
     raise RMT::Mirror::Exception.new(_('Error while mirroring metadata: %{error}') % { error: e.message })
   end
 
-  def mirror_license(repository_dir, repository_url, temp_licenses_dir)
-    mirroring_paths = {
-      base_url: repository_url.chomp('/') + '.license/',
-      base_dir: temp_licenses_dir,
-      cache_dir: repository_dir.chomp('/') + '.license/'
-    }
-
-    begin
-      directory_yast = RMT::Mirror::FileReference.new(relative_path: 'directory.yast', **mirroring_paths)
-      downloader.download_multi([directory_yast])
-    rescue RMT::Downloader::Exception
-      logger.debug("No license directory found for repository '#{repository_url}'")
-      FileUtils.remove_entry(temp_licenses_dir) # the repository would have an empty licenses directory unless removed
-      return
-    end
-
-    license_files = File.readlines(directory_yast.local_path)
-      .map(&:strip).reject { |item| item == 'directory.yast' }
-      .map { |relative_path| RMT::Mirror::FileReference.new(relative_path: relative_path, **mirroring_paths) }
-    downloader.download_multi(license_files)
-  rescue StandardError => e
-    raise RMT::Mirror::Exception.new(_('Error while mirroring license files: %{error}') % { error: e.message })
-  end
-
-  def mirror_packages(metadata_files, repository_dir, repository_url)
-    package_references = parse_packages_metadata(metadata_files)
+  def mirror_packages(metadata_references)
+    package_references = parse_packages_metadata(metadata_references)
 
-    package_file_references = package_references.map do |reference|
+    packages = package_references.map do |reference|
       RMT::Mirror::FileReference.build_from_metadata(reference,
-                                                     base_dir: repository_dir,
+                                                     base_dir: repository_path,
                                                      base_url: repository_url)
     end
 
-    failed_downloads = download_package_files(package_file_references)
+    packages.each do |package|
+      enqueue package if need_to_download?(package)
+    end
+
+    failed = download_enqueued(continue_on_error: true)
 
-    raise _('Failed to download %{failed_count} files') % { failed_count: failed_downloads.size } unless failed_downloads.empty?
+    raise _('Failed to download %{failed_count} files') % { failed_count: failed.size } unless failed.empty?
   rescue StandardError => e
     raise RMT::Mirror::Exception.new(_('Error while mirroring packages: %{error}') % { error: e.message })
   end
@@ -146,34 +66,4 @@ def parse_packages_metadata(metadata_references)
       .map { |file| xml_parsers[file.type]&.new(file.local_path) }.compact
       .map(&:parse).flatten
   end
-
-  def download_package_files(file_references)
-    files_to_download = file_references.select { |file| need_to_download?(file) }
-    return [] if files_to_download.empty?
-
-    downloader.download_multi(files_to_download, ignore_errors: true)
-  end
-
-  def need_to_download?(file)
-    return false if file.arch == 'src' && !mirror_src
-    return false if validate_local_file(file)
-    return false if deduplicate(file)
-
-    true
-  end
-
-  def replace_directory(source_dir, destination_dir)
-    old_directory = File.join(File.dirname(destination_dir), '.old_' + File.basename(destination_dir))
-
-    FileUtils.remove_entry(old_directory) if Dir.exist?(old_directory)
-    FileUtils.mv(destination_dir, old_directory) if Dir.exist?(destination_dir)
-    FileUtils.mv(source_dir, destination_dir, force: true)
-    FileUtils.chmod(0o755, destination_dir)
-  rescue StandardError => e
-    raise RMT::Mirror::Exception.new(_('Error while moving directory %{src} to %{dest}: %{error}') % {
-      src: source_dir,
-      dest: destination_dir,
-      error: e.message
-    })
-  end
 end

spec/fixtures/files/directory.yast

@@ -0,0 +1,12 @@
+directory.yast
+license.de.txt
+license.es.txt
+license.fr.txt
+license.it.txt
+license.ja.txt
+license.ko.txt
+license.pt_BR.txt
+license.ru.txt
+license.txt
+license.zh_CN.txt
+license.zh_TW.txt