[1/x] Prepare code sprint and implement repository type detection

.rubocop_todo.yml

@@ -16,7 +16,7 @@ Layout/ArgumentAlignment:
     - 'app/models/product.rb'
     - 'engines/instance_verification/spec/requests/api/connect/v3/systems/products_controller_spec.rb'
     - 'spec/factories/products.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
+    - 'spec/lib/rmt/mirror/repomd_spec.rb'
     - 'spec/models/migration_engine_spec.rb'
 
 # Offense count: 3
@@ -40,7 +40,7 @@ Layout/EmptyLineAfterGuardClause:
     - 'lib/rmt/cli/repos_custom.rb'
     - 'lib/rmt/cli/smt_importer.rb'
     - 'lib/rmt/fiber_request.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
+    - 'spec/lib/rmt/mirror/repomd_spec.rb'
 
 # Offense count: 2
 # Cop supports --auto-correct.
@@ -241,15 +241,15 @@ Performance/RegexpMatch:
     - 'engines/registration_sharing/lib/registration_sharing.rb'
     - 'engines/strict_authentication/app/controllers/strict_authentication/authentication_controller.rb'
     - 'lib/rmt/cli/base.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
+    - 'spec/lib/rmt/mirror/repomd_spec.rb'
 
 # Offense count: 4
 # Cop supports --auto-correct.
 # Configuration parameters: AutoCorrect.
 Performance/StringInclude:
   Exclude:
     - 'engines/strict_authentication/app/controllers/strict_authentication/authentication_controller.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
+    - 'spec/lib/rmt/mirror/repomd_spec.rb'
 
 # Offense count: 167
 # Configuration parameters: Prefixes.
@@ -279,7 +279,7 @@ RSpec/EmptyLineAfterHook:
     - 'engines/strict_authentication/spec/requests/services_controller_spec.rb'
     - 'engines/strict_authentication/spec/requests/strict_authentication/authentication_controller_spec.rb'
     - 'spec/lib/rmt/config_spec.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
+    - 'spec/lib/rmt/mirror/repomd_spec.rb'
     - 'spec/models/product_spec.rb'
     - 'spec/requests/api/connect/v3/subscriptions/systems_controller_spec.rb'
     - 'spec/requests/api/connect/v3/systems/activations_controller_spec.rb'
@@ -321,7 +321,7 @@ RSpec/ExpectInHook:
     - 'spec/lib/rmt/cli/main_spec.rb'
     - 'spec/lib/rmt/cli/repos_custom_spec.rb'
     - 'spec/lib/rmt/cli/repos_spec.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
+    - 'spec/lib/rmt/mirror/repomd_spec.rb'
     - 'spec/lib/rmt/scc_spec.rb'
 
 # Offense count: 3
@@ -362,7 +362,7 @@ RSpec/ImplicitSubject:
 # Configuration parameters: AssignmentOnly.
 RSpec/InstanceVariable:
   Exclude:
-    - 'spec/lib/rmt/mirror_spec.rb'
+    - 'spec/lib/rmt/mirror/repomd_spec.rb'
 
 # Offense count: 1
 RSpec/IteratedExpectation:
@@ -430,7 +430,7 @@ RSpec/Rails/HttpStatus:
 RSpec/ReceiveCounts:
   Exclude:
     - 'spec/lib/rmt/lockfile_spec.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
+    - 'spec/lib/rmt/mirror/repomd_spec.rb'
     - 'spec/suse/connect/api_spec.rb'
 
 # Offense count: 3
@@ -454,7 +454,7 @@ RSpec/StubbedMock:
     - 'spec/lib/rmt/cli/main_spec.rb'
     - 'spec/lib/rmt/cli/smt_importer_spec.rb'
     - 'spec/lib/rmt/cli/systems_spec.rb'
-    - 'spec/lib/rmt/mirror_spec.rb'
+    - 'spec/lib/rmt/mirror/repomd_spec.rb'
     - 'spec/lib/rmt/scc_spec.rb'
     - 'spec/support/shared_examples/cli.rb'
     - 'spec/suse/connect/api_spec.rb'

lib/rmt/cli/export.rb

@@ -30,7 +30,7 @@ def settings(path)
   def repos(path)
     path = needs_path(path, writable: true)
 
-    mirror = RMT::Mirror.new(mirroring_base_dir: path, logger: logger, airgap_mode: true)
+    mirror = RMT::Mirror::Repomd.new(mirroring_base_dir: path, logger: logger, airgap_mode: true)
 
     begin
       mirror.mirror_suma_product_tree(repository_url: 'https://scc.suse.com/suma/')

lib/rmt/cli/import.rb

@@ -13,7 +13,7 @@ def repos(path)
     RMT::Lockfile.lock do
       path = needs_path(path)
 
-      mirror = RMT::Mirror.new(logger: logger, airgap_mode: true)
+      mirror = RMT::Mirror::Repomd.new(logger: logger, airgap_mode: true)
 
       repos_file = File.join(path, 'repos.json')
       raise RMT::CLI::Error.new(_('%{file} does not exist.') % { file: repos_file }) unless File.exist?(repos_file)

lib/rmt/cli/mirror.rb

@@ -79,7 +79,13 @@ def product(*targets)
   protected
 
   def mirror
-    @mirror ||= RMT::Mirror.new(logger: logger, mirror_src: RMT::Config.mirror_src_files?)
+    config = {
+      logger: logger,
+      mirroring_base_dir: RMT::DEFAULT_MIRROR_DIR,
+      mirror_src: RMT::Config.mirror_src_files?,
+      airgap_mode: false
+    }
+    @mirror ||= RMT::Mirror::Repomd.new(**config)
   end
 
   def errors

lib/rmt/mirror.rb

@@ -1,205 +1,38 @@
-require 'rmt/downloader'
-require 'rmt/gpg'
-require 'repomd_parser'
-require 'time'
-
 class RMT::Mirror
-  class RMT::Mirror::Exception < RuntimeError
-  end
+  RPM_FILE_NEEDLE = 'repodata/repomd.xml'.freeze
+  DEB_FILE_NEEDLE = 'Release'.freeze
 
-  include RMT::Deduplicator
-  include RMT::FileValidator
+  attr_reader :logger, :base_dir, :mirror_sources, :is_airgapped, :repository
 
-  def initialize(mirroring_base_dir: RMT::DEFAULT_MIRROR_DIR, logger:, mirror_src: false, airgap_mode: false)
-    @mirroring_base_dir = mirroring_base_dir
+  def initialize(repository:, base_dir:, logger:, mirror_sources: false, is_airgapped: false)
+    @repository = repository
     @logger = logger
-    @mirror_src = mirror_src
-    @airgap_mode = airgap_mode
-    @deep_verify = false
-
-    # don't save files for deduplication when in offline mode
-    @downloader = RMT::Downloader.new(logger: logger, track_files: !airgap_mode)
+    @base_dir = base_dir
+    @mirror_sources = mirror_sources
+    @is_airgapped = is_airgapped
   end
 
-  def mirror_suma_product_tree(repository_url:)
-    # we have an inconsistency in how we mirror in offline mode
-    # in normal mode we mirror in the following way:
-    # base_dir/repo/...
-    # however, in offline mode we mirror in the following way
-    # base_dir/...
-    # we need this extra step to ensure that we write to the public directory
-    base_dir = mirroring_base_dir
-    base_dir = File.expand_path(File.join(mirroring_base_dir, '/../')) if mirroring_base_dir == RMT::DEFAULT_MIRROR_DIR
-
-    repository_dir = File.join(base_dir, '/suma/')
-    mirroring_paths = {
-      base_url: URI.join(repository_url),
-      base_dir: repository_dir,
-      cache_dir: repository_dir
+  def detect_repository_type
+    search = {
+      repomd: File.join(repository.external_url, RPM_FILE_NEEDLE),
+      debian: File.join(repository.external_url, DEB_FILE_NEEDLE)
     }
 
-    logger.info _('Mirroring SUSE Manager product tree to %{dir}') % { dir: repository_dir }
-    downloader.download_multi([FileReference.new(relative_path: 'product_tree.json', **mirroring_paths)])
-  rescue RMT::Downloader::Exception => e
-    raise RMT::Mirror::Exception.new(_('Could not mirror SUSE Manager product tree with error: %{error}') % { error: e.message })
-  end
-
-  def mirror(repository_url:, local_path:, auth_token: nil, repo_name: nil)
-    repository_dir = File.join(mirroring_base_dir, local_path)
-
-    logger.info _('Mirroring repository %{repo} to %{dir}') % { repo: repo_name || repository_url, dir: repository_dir }
-
-    create_repository_dir(repository_dir)
-    temp_licenses_dir = create_temp_dir
-    # downloading license doesn't require an auth token
-    mirror_license(repository_dir, repository_url, temp_licenses_dir)
-
-    downloader.auth_token = auth_token
-    temp_metadata_dir = create_temp_dir
-    metadata_files = mirror_metadata(repository_dir, repository_url, temp_metadata_dir)
-    mirror_packages(metadata_files, repository_dir, repository_url)
-
-    replace_directory(temp_licenses_dir, repository_dir.chomp('/') + '.license/') if Dir.exist?(temp_licenses_dir)
-    replace_directory(File.join(temp_metadata_dir, 'repodata'), File.join(repository_dir, 'repodata'))
-  ensure
-    [temp_licenses_dir, temp_metadata_dir].each { |dir| FileUtils.remove_entry(dir, true) }
-  end
-
-  protected
-
-  attr_reader :airgap_mode, :deep_verify, :downloader, :logger, :mirroring_base_dir, :mirror_src
-
-  def create_repository_dir(repository_dir)
-    FileUtils.mkpath(repository_dir) unless Dir.exist?(repository_dir)
-  rescue StandardError => e
-    raise RMT::Mirror::Exception.new(
-      _('Could not create local directory %{dir} with error: %{error}') % { dir: repository_dir, error: e.message }
-    )
-  end
-
-  def create_temp_dir
-    Dir.mktmpdir
-  rescue StandardError => e
-    raise RMT::Mirror::Exception.new(_('Could not create a temporary directory: %{error}') % { error: e.message })
-  end
-
-  def mirror_metadata(repository_dir, repository_url, temp_metadata_dir)
-    mirroring_paths = {
-      base_url: URI.join(repository_url),
-      base_dir: temp_metadata_dir,
-      cache_dir: repository_dir
-    }
-
-    repomd_xml = FileReference.new(relative_path: 'repodata/repomd.xml', **mirroring_paths)
-    downloader.download_multi([repomd_xml])
-
-    begin
-      signature_file = FileReference.new(relative_path: 'repodata/repomd.xml.asc', **mirroring_paths)
-      key_file       = FileReference.new(relative_path: 'repodata/repomd.xml.key', **mirroring_paths)
-      # mirror repomd.xml.asc first, because there are repos with repomd.xml.asc but without repomd.xml.key
-      downloader.download_multi([signature_file])
-      downloader.download_multi([key_file])
-
-      RMT::GPG.new(
-        metadata_file: repomd_xml.local_path,
-        key_file: key_file.local_path,
-        signature_file: signature_file.local_path,
-        logger: logger
-      ).verify_signature
-    rescue RMT::Downloader::Exception => e
-      if (e.http_code == 404)
-        logger.info(_('Repository metadata signatures are missing'))
-      else
-        raise(_('Downloading repo signature/key failed with: %{message}, HTTP code %{http_code}') % { message: e.message, http_code: e.http_code })
+    search.each do |key, url|
+      # Current CDN authenticates via a key append to the request path
+      # e.g.
+      # https://update.suse.com/SUSE/product/some-product
+      # becomes
+      # https://update.suse.com/SUSE/product/some-product?authenication_tokensiduhashasdyashdaysdasud
+      uri = URI.join(url)
+      uri.query = @repository.auth_token if @repository.auth_token
+
+      request = RMT::HttpRequest.new(uri, method: :head, followlocation: true)
+      request.on_success do
+        return key
       end
+      request.run
     end
-
-    metadata_files = RepomdParser::RepomdXmlParser.new(repomd_xml.local_path).parse
-      .map { |reference| FileReference.build_from_metadata(reference, **mirroring_paths) }
-
-    downloader.download_multi(metadata_files.dup)
-
-    metadata_files
-  rescue StandardError => e
-    raise RMT::Mirror::Exception.new(_('Error while mirroring metadata: %{error}') % { error: e.message })
-  end
-
-  def mirror_license(repository_dir, repository_url, temp_licenses_dir)
-    mirroring_paths = {
-      base_url: repository_url.chomp('/') + '.license/',
-      base_dir: temp_licenses_dir,
-      cache_dir: repository_dir.chomp('/') + '.license/'
-    }
-
-    begin
-      directory_yast = FileReference.new(relative_path: 'directory.yast', **mirroring_paths)
-      downloader.download_multi([directory_yast])
-    rescue RMT::Downloader::Exception
-      logger.debug("No license directory found for repository '#{repository_url}'")
-      FileUtils.remove_entry(temp_licenses_dir) # the repository would have an empty licenses directory unless removed
-      return
-    end
-
-    license_files = File.readlines(directory_yast.local_path)
-      .map(&:strip).reject { |item| item == 'directory.yast' }
-      .map { |relative_path| FileReference.new(relative_path: relative_path, **mirroring_paths) }
-    downloader.download_multi(license_files)
-  rescue StandardError => e
-    raise RMT::Mirror::Exception.new(_('Error while mirroring license files: %{error}') % { error: e.message })
-  end
-
-  def mirror_packages(metadata_files, repository_dir, repository_url)
-    package_references = parse_packages_metadata(metadata_files)
-
-    package_file_references = package_references.map do |reference|
-      FileReference.build_from_metadata(reference,
-                                        base_dir: repository_dir,
-                                        base_url: repository_url)
-    end
-
-    failed_downloads = download_package_files(package_file_references)
-
-    raise _('Failed to download %{failed_count} files') % { failed_count: failed_downloads.size } unless failed_downloads.empty?
-  rescue StandardError => e
-    raise RMT::Mirror::Exception.new(_('Error while mirroring packages: %{error}') % { error: e.message })
-  end
-
-  def parse_packages_metadata(metadata_references)
-    xml_parsers = { deltainfo: RepomdParser::DeltainfoXmlParser,
-                    primary: RepomdParser::PrimaryXmlParser }
-
-    metadata_references
-      .map { |file| xml_parsers[file.type]&.new(file.local_path) }.compact
-      .map(&:parse).flatten
-  end
-
-  def download_package_files(file_references)
-    files_to_download = file_references.select { |file| need_to_download?(file) }
-    return [] if files_to_download.empty?
-
-    downloader.download_multi(files_to_download, ignore_errors: true)
-  end
-
-  def need_to_download?(file)
-    return false if file.arch == 'src' && !mirror_src
-    return false if validate_local_file(file)
-    return false if deduplicate(file)
-
-    true
-  end
-
-  def replace_directory(source_dir, destination_dir)
-    old_directory = File.join(File.dirname(destination_dir), '.old_' + File.basename(destination_dir))
-
-    FileUtils.remove_entry(old_directory) if Dir.exist?(old_directory)
-    FileUtils.mv(destination_dir, old_directory) if Dir.exist?(destination_dir)
-    FileUtils.mv(source_dir, destination_dir, force: true)
-    FileUtils.chmod(0o755, destination_dir)
-  rescue StandardError => e
-    raise RMT::Mirror::Exception.new(_('Error while moving directory %{src} to %{dest}: %{error}') % {
-      src: source_dir,
-      dest: destination_dir,
-      error: e.message
-    })
+    nil
   end
 end

lib/rmt/mirror/exception.rb

@@ -0,0 +1,2 @@
+class RMT::Mirror::Exception < RuntimeError
+end