Merge pull request #456 from SUSE/mirror-individual-repos

Mirror individual products/repos

.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2019-07-04 10:28:23 +0200 using RuboCop version 0.53.0.
+# on 2019-08-13 14:02:20 +0200 using RuboCop version 0.53.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -11,7 +11,7 @@ FactoryGirl/DynamicAttributeDefinedStatically:
   Exclude:
     - 'spec/factories/subscriptions.rb'
 
-# Offense count: 125
+# Offense count: 126
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle.
 # SupportedStyles: empty_lines, empty_lines_except_namespace, empty_lines_special, no_empty_lines, beginning_only, ending_only
@@ -29,7 +29,7 @@ Lint/MissingCopEnableDirective:
 # Offense count: 2
 # Configuration parameters: CountComments.
 Metrics/ClassLength:
-  Max: 167
+  Max: 165
 
 # Offense count: 1
 # Configuration parameters: CountComments.
@@ -48,7 +48,7 @@ RSpec/ExampleLength:
   Exclude:
     - 'spec/features/**/*'
     - 'spec/lib/rmt/cli/import_spec.rb'
-    - 'spec/lib/rmt/cli/main_spec.rb'
+    - 'spec/lib/rmt/cli/mirror_spec.rb'
     - 'spec/lib/rmt/rpm/repomd_xml_parser_spec.rb'
 
 # Offense count: 60
@@ -153,11 +153,6 @@ Rails/LexicallyScopedActionFilter:
   Exclude:
     - 'app/controllers/api/connect/v3/systems/products_controller.rb'
 
-# Offense count: 1
-Style/DateTime:
-  Exclude:
-    - 'spec/lib/rmt/cli/main_spec.rb'
-
 # Offense count: 6
 # Cop supports --auto-correct.
 Style/ExpandPathArguments:

app/models/product.rb

@@ -144,4 +144,21 @@ def create_service!
 
     service
   end
+
+  def self.get_by_target!(target)
+    product_id = Integer(target, 10) rescue nil
+
+    products = []
+    if product_id
+      product = find(product_id)
+      products << product unless product.nil?
+    else
+      identifier, version, arch = target.split('/')
+      conditions = { identifier: identifier, version: version }
+      conditions[:arch] = arch if arch
+      products = where(conditions).to_a
+    end
+
+    products
+  end
 end

lib/rmt.rb

@@ -1,5 +1,5 @@
 module RMT
-  VERSION ||= '2.3.7'.freeze
+  VERSION ||= '2.4.0'.freeze
 
   DEFAULT_USER = '_rmt'.freeze
   DEFAULT_GROUP = 'nginx'.freeze

lib/rmt/cli/main.rb

@@ -18,40 +18,7 @@ def sync
   subcommand 'repos', RMT::CLI::Repos
 
   desc 'mirror', _('Mirror repositories')
-  def mirror
-    RMT::Lockfile.lock do
-      logger = RMT::Logger.new(STDOUT)
-      mirror = RMT::Mirror.new(logger: logger, mirror_src: RMT::Config.mirror_src_files?)
-
-      begin
-        mirror.mirror_suma_product_tree(repository_url: 'https://scc.suse.com/suma/')
-      rescue RMT::Mirror::Exception => e
-        logger.warn(e.message)
-      end
-
-      raise RMT::CLI::Error.new(_('There are no repositories marked for mirroring.')) if Repository.where(mirroring_enabled: true).empty?
-
-      mirrored_repo_ids = []
-      until Repository.where(mirroring_enabled: true).where.not(id: mirrored_repo_ids).blank?
-        repo = Repository.where(mirroring_enabled: true).where.not(id: mirrored_repo_ids).first
-
-        begin
-          mirror.mirror(
-            repository_url: repo.external_url,
-            local_path: Repository.make_local_path(repo.external_url),
-            auth_token: repo.auth_token,
-            repo_name: repo.name
-          )
-
-          repo.refresh_timestamp!
-        rescue RMT::Mirror::Exception => e
-          logger.warn e.to_s
-        ensure
-          mirrored_repo_ids << repo.id
-        end
-      end
-    end
-  end
+  subcommand 'mirror', RMT::CLI::Mirror
 
   desc 'import', _('Import commands for Offline Sync')
   subcommand 'import', RMT::CLI::Import

lib/rmt/cli/mirror.rb

@@ -0,0 +1,103 @@
+class RMT::CLI::Mirror < RMT::CLI::Base
+  desc 'all', _('Mirror all enabled repositories')
+  def all
+    RMT::Lockfile.lock do
+      logger = RMT::Logger.new(STDOUT)
+      mirror = RMT::Mirror.new(logger: logger, mirror_src: RMT::Config.mirror_src_files?)
+
+      begin
+        mirror.mirror_suma_product_tree(repository_url: 'https://scc.suse.com/suma/')
+      rescue RMT::Mirror::Exception => e
+        logger.warn(e.message)
+      end
+
+      raise RMT::CLI::Error.new(_('There are no repositories marked for mirroring.')) if Repository.where(mirroring_enabled: true).empty?
+
+      mirrored_repo_ids = []
+      until Repository.where(mirroring_enabled: true).where.not(id: mirrored_repo_ids).blank?
+        repo = Repository.where(mirroring_enabled: true).where.not(id: mirrored_repo_ids).first
+
+        begin
+          mirror_repo!(mirror, repo)
+        rescue RMT::Mirror::Exception => e
+          logger.warn e.to_s
+        ensure
+          mirrored_repo_ids << repo.id
+        end
+      end
+    end
+  end
+
+  default_task :all
+
+  desc 'repository IDS', _('Mirror enabled repositories with given repository IDs')
+  def repository(*ids)
+    RMT::Lockfile.lock do
+      logger = RMT::Logger.new(STDOUT)
+      mirror = RMT::Mirror.new(logger: logger, mirror_src: RMT::Config.mirror_src_files?)
+
+      ids = clean_target_input(ids)
+      raise RMT::CLI::Error.new(_('No repository IDs supplied')) if ids.empty?
+
+      repos = []
+      ids.each do |id|
+        repo = Repository.find_by!(scc_id: id)
+        raise RMT::CLI::Error.new(_('Mirroring of repository with ID %{repo_id} is not enabled') % { repo_id: id }) unless repo.mirroring_enabled
+        repos << repo
+      rescue ActiveRecord::RecordNotFound
+        raise RMT::CLI::Error.new(_('Repository with ID %{repo_id} not found') % { repo_id: id })
+      end
+
+      repos.each do |repo|
+        mirror_repo!(mirror, repo)
+      rescue RMT::Mirror::Exception => e
+        logger.warn e.to_s
+      end
+    end
+  end
+
+  desc 'product IDS', _('Mirror enabled repositories for a product with given product IDs')
+  def product(*targets)
+    RMT::Lockfile.lock do
+      logger = RMT::Logger.new(STDOUT)
+      mirror = RMT::Mirror.new(logger: logger, mirror_src: RMT::Config.mirror_src_files?)
+
+      targets = clean_target_input(targets)
+      raise RMT::CLI::Error.new(_('No product IDs supplied')) if targets.empty?
+
+      repos = []
+      targets.each do |target|
+        products = Product.get_by_target!(target)
+        raise RMT::CLI::Error.new(_('Product for target %{target} not found') % { target: target }) if products.empty?
+        products.each do |product|
+          product_repos = product.repositories.where(mirroring_enabled: true)
+          raise RMT::CLI::Error.new(_('Product %{target} has no repositories enabled') % { target: target }) if product_repos.empty?
+          repos += product_repos.to_a
+        end
+      rescue ActiveRecord::RecordNotFound
+        raise RMT::CLI::Error.new(_('Product with ID %{target} not found') % { target: target })
+      end
+
+      repos.each do |repo|
+        mirror_repo!(mirror, repo)
+      rescue RMT::Mirror::Exception => e
+        logger.warn e.to_s
+      end
+    end
+  end
+
+  protected
+
+  def mirror_repo!(mirror, repo)
+    mirror.mirror(
+      repository_url: repo.external_url,
+      local_path: Repository.make_local_path(repo.external_url),
+      auth_token: repo.auth_token,
+      repo_name: repo.name
+    )
+
+    repo.refresh_timestamp!
+  end
+
+
+end

lib/rmt/cli/products.rb

@@ -144,22 +144,9 @@ def change_product(target, set_enabled, all_modules)
   private
 
   def find_products(target)
-    product_id = Integer(target, 10) rescue nil
-
-    products = []
-    if product_id
-      product = Product.find(product_id)
-      products << product unless product.nil?
-    else
-      identifier, version, arch = target.split('/')
-      conditions = { identifier: identifier, version: version }
-      conditions[:arch] = arch if arch
-      products = Product.where(conditions).to_a
-    end
-
-    products
+    Product.get_by_target!(target)
   rescue ActiveRecord::RecordNotFound
-    raise ProductNotFoundException.new(_('Product by ID %{id} not found.') % { id: product_id })
+    raise ProductNotFoundException.new(_('Product by ID %{id} not found.') % { id: target })
   end
 
   def repository_service

package/obs/rmt-server.changes

@@ -1,3 +1,9 @@
+-------------------------------------------------------------------
+Tue Aug 20 11:12:58 UTC 2019 - Ivan Kapelyukhin <[email protected]>
+
+- Version 2.4.0
+- Allow to mirror individual products/repositories (bsc#1138863)
+
 -------------------------------------------------------------------
 Fri Aug  16 11:07:26 UTC 2019 - [email protected]
 

package/obs/rmt-server.spec

@@ -25,7 +25,7 @@
 %define ruby_version %{rb_default_ruby_suffix}
 
 Name:           rmt-server
-Version:        2.3.7
+Version:        2.4.0
 Release:        0
 Summary:        Repository mirroring tool and registration proxy for SCC
 License:        GPL-2.0-or-later

spec/lib/rmt/cli/main_spec.rb

@@ -21,118 +21,6 @@
       end
     end
 
-    describe 'mirror' do
-      let(:argv) { ['mirror'] }
-
-      include_examples 'handles lockfile exception'
-
-      context 'suma product tree mirror with exception' do
-        before do
-          create :repository, :with_products, mirroring_enabled: true
-        end
-
-        it 'outputs exception message' do
-          expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree).and_raise(RMT::Mirror::Exception, 'black mirror')
-          expect_any_instance_of(RMT::Mirror).to receive(:mirror)
-          expect_any_instance_of(RMT::Logger).to receive(:warn).with('black mirror')
-          command
-        end
-      end
-
-      context 'without repositories marked for mirroring' do
-        before do
-          create :repository, :with_products, mirroring_enabled: false
-        end
-
-        it 'outputs a warning' do
-          expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree)
-          expect_any_instance_of(RMT::Mirror).not_to receive(:mirror)
-          expect { command }.to raise_error(SystemExit).and output("There are no repositories marked for mirroring.\n").to_stderr.and output('').to_stdout
-        end
-      end
-
-      context 'with repositories marked for mirroring' do
-        let!(:repository) { create :repository, :with_products, mirroring_enabled: true }
-
-        it 'updates repository mirroring timestamp' do
-          expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree)
-          expect_any_instance_of(RMT::Mirror).to receive(:mirror)
-
-          Timecop.freeze(Time.utc(2018)) do
-            expect { command }.to change { repository.reload.last_mirrored_at }.to(DateTime.now.utc)
-          end
-        end
-
-        context 'with exceptions during mirroring' do
-          before { allow_any_instance_of(RMT::Mirror).to receive(:mirror).and_raise(RMT::Mirror::Exception, 'black mirror') }
-
-          it 'outputs exception message' do
-            expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree)
-            expect_any_instance_of(RMT::Logger).to receive(:warn).with('black mirror')
-            command
-          end
-        end
-      end
-
-      context 'with repositories changing during mirroring' do
-        let!(:repository) { create :repository, :with_products, mirroring_enabled: true }
-        let!(:additional_repository) { create :repository, :with_products, mirroring_enabled: false }
-
-        it 'mirrors additional repositories' do
-          expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree)
-          expect_any_instance_of(RMT::Mirror).to receive(:mirror).with(
-            repository_url: repository.external_url,
-            local_path: anything,
-            repo_name: anything,
-            auth_token: anything
-          ) do
-            # enable mirroring of the additional repository during mirroring
-            additional_repository.mirroring_enabled = true
-            additional_repository.save!
-          end
-
-          expect_any_instance_of(RMT::Mirror).to receive(:mirror).with(
-            repository_url: additional_repository.external_url,
-            local_path: anything,
-            repo_name: anything,
-            auth_token: anything
-          )
-
-          command
-        end
-      end
-
-      context 'with repositories changing during mirroring and exceptions occur' do
-        let!(:repository) { create :repository, :with_products, mirroring_enabled: true }
-        let!(:additional_repository) { create :repository, :with_products, mirroring_enabled: false }
-
-        it 'handles exceptions and mirrors additional repositories' do
-          expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree)
-          expect_any_instance_of(RMT::Mirror).to receive(:mirror).with(
-            repository_url: repository.external_url,
-            local_path: anything,
-            repo_name: anything,
-            auth_token: anything
-          ) do
-            # enable mirroring of the additional repository during mirroring
-            additional_repository.mirroring_enabled = true
-            additional_repository.save!
-            raise(RMT::Mirror::Exception, 'black mirror')
-          end
-
-          expect_any_instance_of(RMT::Logger).to receive(:warn).with('black mirror')
-          expect_any_instance_of(RMT::Mirror).to receive(:mirror).with(
-            repository_url: additional_repository.external_url,
-            local_path: anything,
-            repo_name: anything,
-            auth_token: anything
-          )
-
-          command
-        end
-      end
-    end
-
     describe 'help' do
       let(:argv) { ['help'] }