From c08849b29aa9f7bf562703ebf17fc6dcfafd9e89 Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Sun, 14 Jul 2019 12:46:47 +0000 Subject: [PATCH 01/11] Update active_model_serializers to version 0.10.10 --- Gemfile.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 1d2a184c1..3035e511f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -14,9 +14,9 @@ GEM erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.3) - active_model_serializers (0.10.9) - actionpack (>= 4.1, < 6) - activemodel (>= 4.1, < 6) + active_model_serializers (0.10.10) + actionpack (>= 4.1, < 6.1) + activemodel (>= 4.1, < 6.1) case_transform (>= 0.2) jsonapi-renderer (>= 0.1.1.beta1, < 0.3) activemodel (5.1.6.2) @@ -155,7 +155,7 @@ GEM i18n (1.6.0) concurrent-ruby (~> 1.0) json (2.2.0) - jsonapi-renderer (0.2.0) + jsonapi-renderer (0.2.2) jwt (2.2.1) kramdown (2.1.0) kramdown-parser-gfm (1.1.0) From 9a4c53939c0d9541a440b28d20290b113f9b6af6 Mon Sep 17 00:00:00 2001 From: Ivan Kapelyukhin Date: Thu, 8 Aug 2019 12:32:10 +0200 Subject: [PATCH 02/11] Make Rails log level configurable; add log_level to default config --- config/environments/production.rb | 8 +++++++- package/obs/rmt.conf | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index 734311a0d..398cf148e 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -19,7 +19,13 @@ # Use the lowest log level to ensure availability of diagnostic information # when problems arise. - config.log_level = :debug + + log_level = Settings&.log_level&.rails&.to_s&.upcase + config.log_level = if RMT::Logger::LABEL_TO_SEVERITY_MAPPING[log_level] + log_level + else + :debug + end # Prepend all log lines with the following tags. config.log_tags = [ :request_id ] diff --git a/package/obs/rmt.conf b/package/obs/rmt.conf index 609a8f5b2..72726a73a 100644 --- a/package/obs/rmt.conf +++ b/package/obs/rmt.conf @@ -23,3 +23,6 @@ http_client: proxy_auth: proxy_user: proxy_password: + +log_level: + rails: info From 544cc8abc7d7c73aeae0435105919d6222f49dcd Mon Sep 17 00:00:00 2001 From: Ivan Kapelyukhin Date: Thu, 8 Aug 2019 12:51:31 +0200 Subject: [PATCH 03/11] Bump the version; add changelog entry --- lib/rmt.rb | 2 +- package/obs/rmt-server.changes | 6 ++++++ package/obs/rmt-server.spec | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/rmt.rb b/lib/rmt.rb index 90c3fadec..5bc7bef20 100644 --- a/lib/rmt.rb +++ b/lib/rmt.rb @@ -1,5 +1,5 @@ module RMT - VERSION ||= '2.3.5'.freeze + VERSION ||= '2.3.6'.freeze DEFAULT_USER = '_rmt'.freeze DEFAULT_GROUP = 'nginx'.freeze diff --git a/package/obs/rmt-server.changes b/package/obs/rmt-server.changes index 3f85463ec..5548e2379 100644 --- a/package/obs/rmt-server.changes +++ b/package/obs/rmt-server.changes @@ -1,3 +1,9 @@ +------------------------------------------------------------------- +Thu Aug 8 10:50:26 UTC 2019 - Ivan Kapelyukhin + +- Version 2.3.6 +- Make Rails log level configurable; set default log level to INFO + ------------------------------------------------------------------- Wed Aug 7 12:43:02 UTC 2019 - Thomas Schmidt diff --git a/package/obs/rmt-server.spec b/package/obs/rmt-server.spec index 9e47840af..04e31b657 100644 --- a/package/obs/rmt-server.spec +++ b/package/obs/rmt-server.spec @@ -25,7 +25,7 @@ %define ruby_version %{rb_default_ruby_suffix} Name: rmt-server -Version: 2.3.5 +Version: 2.3.6 Release: 0 Summary: Repository mirroring tool and registration proxy for SCC License: GPL-2.0-or-later From f380e84322bddfdf89c386673e810cc0f6915660 Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2019 08:39:35 +0000 Subject: [PATCH 04/11] Update nokogiri to version 1.10.4 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 666dfa2f8..739410273 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -180,7 +180,7 @@ GEM nap (1.1.0) nenv (0.3.0) no_proxy_fix (0.1.2) - nokogiri (1.10.3) + nokogiri (1.10.4) mini_portile2 (~> 2.4.0) notiffany (0.1.1) nenv (~> 0.1) From 4e35fe4dcaa42d830dee30c11168cc9dc8325aee Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2019 13:09:01 +0000 Subject: [PATCH 05/11] Update shoulda-matchers to version 4.1.2 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 739410273..e7e894d15 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -274,7 +274,7 @@ GEM rubocop-rspec (= 1.19) sexp_processor (4.12.0) shellany (0.0.1) - shoulda-matchers (4.1.1) + shoulda-matchers (4.1.2) activesupport (>= 4.2.0) simplecov (0.16.1) docile (~> 1.1) From 55a2b24c73b7f6df5a3d6a4d7d9dedf14ad3628e Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2019 13:18:30 +0000 Subject: [PATCH 06/11] Update webmock to version 3.6.2 --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index e7e894d15..274c503f7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -150,7 +150,7 @@ GEM guard (~> 2.1) guard-compat (~> 1.1) rspec (>= 2.99.0, < 4.0) - hashdiff (0.4.0) + hashdiff (1.0.0) hpricot (0.8.6) i18n (1.6.0) concurrent-ruby (~> 1.0) @@ -195,7 +195,7 @@ GEM pry (0.12.2) coderay (~> 1.1.0) method_source (~> 0.9.0) - public_suffix (3.1.0) + public_suffix (3.1.1) puma (3.10.0) rack (2.0.7) rack-test (1.1.0) @@ -306,7 +306,7 @@ GEM activesupport (>= 3) railties (>= 3) yard (~> 0.9.20) - webmock (3.6.0) + webmock (3.6.2) addressable (>= 2.3.6) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) From 603a910468951d3241018d058643028dd7095b72 Mon Sep 17 00:00:00 2001 From: Thomas Muntaner Date: Fri, 16 Aug 2019 10:54:06 +0200 Subject: [PATCH 07/11] Fix mirror_src config We took and documented the configuration value, but never actually used it. --- lib/rmt/cli/main.rb | 2 +- lib/rmt/config.rb | 4 ++++ spec/lib/rmt/config_spec.rb | 23 +++++++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/lib/rmt/cli/main.rb b/lib/rmt/cli/main.rb index 96954793d..86008a8bd 100644 --- a/lib/rmt/cli/main.rb +++ b/lib/rmt/cli/main.rb @@ -21,7 +21,7 @@ def sync def mirror RMT::Lockfile.lock do logger = RMT::Logger.new(STDOUT) - mirror = RMT::Mirror.new(logger: logger) + mirror = RMT::Mirror.new(logger: logger, mirror_src: RMT::Config.mirror_src_files?) begin mirror.mirror_suma_product_tree(repository_url: 'https://scc.suse.com/suma/') diff --git a/lib/rmt/config.rb b/lib/rmt/config.rb index 64a296af4..03f19da46 100644 --- a/lib/rmt/config.rb +++ b/lib/rmt/config.rb @@ -32,4 +32,8 @@ def self.deduplication_by_hardlink? Settings.try(:mirroring).try(:dedup_method).to_s.to_sym != :copy end + def self.mirror_src_files? + ActiveModel::Type::Boolean.new.cast(Settings.try(:mirroring).try(:mirror_src)) + end + end diff --git a/spec/lib/rmt/config_spec.rb b/spec/lib/rmt/config_spec.rb index 53efe8481..5080f1c12 100644 --- a/spec/lib/rmt/config_spec.rb +++ b/spec/lib/rmt/config_spec.rb @@ -1,6 +1,29 @@ require 'rails_helper' RSpec.describe RMT::Config do + describe '#mirroring mirror_src' do + context 'defaults' do + [nil, ''].each do |config_provided| + before { Settings['mirroring'].mirror_src = config_provided } + it("defaults when supplied #{config_provided}") { expect(described_class.mirror_src_files?).to be_falsey } + end + end + + context 'true' do + [true, 'true'].each do |config_provided| + before { Settings['mirroring'].mirror_src = config_provided } + it("defaults when supplied #{config_provided}") { expect(described_class.mirror_src_files?).to be_truthy } + end + end + + context 'false' do + [false, 'false'].each do |config_provided| + before { Settings['mirroring'].mirror_src = config_provided } + it("defaults when supplied #{config_provided}") { expect(described_class.mirror_src_files?).to be_falsey } + end + end + end + describe '#mirroring dedup_method' do context 'defaults' do [nil, ''].each do |dedup_method| From eb99a823b8d767042fdcb1e49e3ff144389a5db7 Mon Sep 17 00:00:00 2001 From: Thomas Muntaner Date: Fri, 16 Aug 2019 11:05:02 +0200 Subject: [PATCH 08/11] Bumped to version 2.3.7 --- lib/rmt.rb | 2 +- package/obs/rmt-server.changes | 7 +++++++ package/obs/rmt-server.spec | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/rmt.rb b/lib/rmt.rb index 5bc7bef20..1bdf1dae8 100644 --- a/lib/rmt.rb +++ b/lib/rmt.rb @@ -1,5 +1,5 @@ module RMT - VERSION ||= '2.3.6'.freeze + VERSION ||= '2.3.7'.freeze DEFAULT_USER = '_rmt'.freeze DEFAULT_GROUP = 'nginx'.freeze diff --git a/package/obs/rmt-server.changes b/package/obs/rmt-server.changes index 5548e2379..181b3d58a 100644 --- a/package/obs/rmt-server.changes +++ b/package/obs/rmt-server.changes @@ -1,3 +1,10 @@ +------------------------------------------------------------------- +Fri Aug 16 11:07:26 UTC 2019 - tmuntaner@suse.com + +- Version 2.3.7 +- RMT now uses mirror_src to determine whether or not to download + source packages. (bsc#1145688) + ------------------------------------------------------------------- Thu Aug 8 10:50:26 UTC 2019 - Ivan Kapelyukhin diff --git a/package/obs/rmt-server.spec b/package/obs/rmt-server.spec index 04e31b657..7a5121fbc 100644 --- a/package/obs/rmt-server.spec +++ b/package/obs/rmt-server.spec @@ -25,7 +25,7 @@ %define ruby_version %{rb_default_ruby_suffix} Name: rmt-server -Version: 2.3.6 +Version: 2.3.7 Release: 0 Summary: Repository mirroring tool and registration proxy for SCC License: GPL-2.0-or-later From 61f826154527bc1004687ba2abfb98c2ffe52491 Mon Sep 17 00:00:00 2001 From: Ivan Kapelyukhin Date: Tue, 13 Aug 2019 14:03:53 +0200 Subject: [PATCH 09/11] Allow to mirror individual products/repositories --- .rubocop_todo.yml | 13 +- app/models/product.rb | 17 +++ lib/rmt/cli/main.rb | 35 +---- lib/rmt/cli/mirror.rb | 99 ++++++++++++ lib/rmt/cli/products.rb | 17 +-- spec/lib/rmt/cli/main_spec.rb | 112 -------------- spec/lib/rmt/cli/mirror_spec.rb | 258 ++++++++++++++++++++++++++++++++ 7 files changed, 381 insertions(+), 170 deletions(-) create mode 100644 lib/rmt/cli/mirror.rb create mode 100644 spec/lib/rmt/cli/mirror_spec.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 4fed03367..d62535296 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -1,6 +1,6 @@ # This configuration was generated by # `rubocop --auto-gen-config` -# on 2019-07-04 10:28:23 +0200 using RuboCop version 0.53.0. +# on 2019-08-13 14:02:20 +0200 using RuboCop version 0.53.0. # The point is for the user to remove these configuration records # one by one as the offenses are removed from the code base. # Note that changes in the inspected code, or installation of new @@ -11,7 +11,7 @@ FactoryGirl/DynamicAttributeDefinedStatically: Exclude: - 'spec/factories/subscriptions.rb' -# Offense count: 125 +# Offense count: 126 # Cop supports --auto-correct. # Configuration parameters: EnforcedStyle. # SupportedStyles: empty_lines, empty_lines_except_namespace, empty_lines_special, no_empty_lines, beginning_only, ending_only @@ -29,7 +29,7 @@ Lint/MissingCopEnableDirective: # Offense count: 2 # Configuration parameters: CountComments. Metrics/ClassLength: - Max: 167 + Max: 165 # Offense count: 1 # Configuration parameters: CountComments. @@ -48,7 +48,7 @@ RSpec/ExampleLength: Exclude: - 'spec/features/**/*' - 'spec/lib/rmt/cli/import_spec.rb' - - 'spec/lib/rmt/cli/main_spec.rb' + - 'spec/lib/rmt/cli/mirror_spec.rb' - 'spec/lib/rmt/rpm/repomd_xml_parser_spec.rb' # Offense count: 60 @@ -153,11 +153,6 @@ Rails/LexicallyScopedActionFilter: Exclude: - 'app/controllers/api/connect/v3/systems/products_controller.rb' -# Offense count: 1 -Style/DateTime: - Exclude: - - 'spec/lib/rmt/cli/main_spec.rb' - # Offense count: 6 # Cop supports --auto-correct. Style/ExpandPathArguments: diff --git a/app/models/product.rb b/app/models/product.rb index e01673dc7..4997f9a78 100644 --- a/app/models/product.rb +++ b/app/models/product.rb @@ -144,4 +144,21 @@ def create_service! service end + + def self.get_by_target!(target) + product_id = Integer(target, 10) rescue nil + + products = [] + if product_id + product = find(product_id) + products << product unless product.nil? + else + identifier, version, arch = target.split('/') + conditions = { identifier: identifier, version: version } + conditions[:arch] = arch if arch + products = where(conditions).to_a + end + + products + end end diff --git a/lib/rmt/cli/main.rb b/lib/rmt/cli/main.rb index 86008a8bd..25ed911be 100644 --- a/lib/rmt/cli/main.rb +++ b/lib/rmt/cli/main.rb @@ -18,40 +18,7 @@ def sync subcommand 'repos', RMT::CLI::Repos desc 'mirror', _('Mirror repositories') - def mirror - RMT::Lockfile.lock do - logger = RMT::Logger.new(STDOUT) - mirror = RMT::Mirror.new(logger: logger, mirror_src: RMT::Config.mirror_src_files?) - - begin - mirror.mirror_suma_product_tree(repository_url: 'https://scc.suse.com/suma/') - rescue RMT::Mirror::Exception => e - logger.warn(e.message) - end - - raise RMT::CLI::Error.new(_('There are no repositories marked for mirroring.')) if Repository.where(mirroring_enabled: true).empty? - - mirrored_repo_ids = [] - until Repository.where(mirroring_enabled: true).where.not(id: mirrored_repo_ids).blank? - repo = Repository.where(mirroring_enabled: true).where.not(id: mirrored_repo_ids).first - - begin - mirror.mirror( - repository_url: repo.external_url, - local_path: Repository.make_local_path(repo.external_url), - auth_token: repo.auth_token, - repo_name: repo.name - ) - - repo.refresh_timestamp! - rescue RMT::Mirror::Exception => e - logger.warn e.to_s - ensure - mirrored_repo_ids << repo.id - end - end - end - end + subcommand 'mirror', RMT::CLI::Mirror desc 'import', _('Import commands for Offline Sync') subcommand 'import', RMT::CLI::Import diff --git a/lib/rmt/cli/mirror.rb b/lib/rmt/cli/mirror.rb new file mode 100644 index 000000000..331476d51 --- /dev/null +++ b/lib/rmt/cli/mirror.rb @@ -0,0 +1,99 @@ +class RMT::CLI::Mirror < RMT::CLI::Base + desc 'all', _('Mirror all enabled repositories') + def all + RMT::Lockfile.lock do + logger = RMT::Logger.new(STDOUT) + mirror = RMT::Mirror.new(logger: logger, mirror_src: RMT::Config.mirror_src_files?) + + begin + mirror.mirror_suma_product_tree(repository_url: 'https://scc.suse.com/suma/') + rescue RMT::Mirror::Exception => e + logger.warn(e.message) + end + + raise RMT::CLI::Error.new(_('There are no repositories marked for mirroring.')) if Repository.where(mirroring_enabled: true).empty? + + mirrored_repo_ids = [] + until Repository.where(mirroring_enabled: true).where.not(id: mirrored_repo_ids).blank? + repo = Repository.where(mirroring_enabled: true).where.not(id: mirrored_repo_ids).first + + begin + mirror_repo!(mirror, repo) + rescue RMT::Mirror::Exception => e + logger.warn e.to_s + ensure + mirrored_repo_ids << repo.id + end + end + end + end + + default_task :all + + desc 'repository IDS', _('Mirror enabled repositories with given repository IDs') + def repository(*ids) + RMT::Lockfile.lock do + logger = RMT::Logger.new(STDOUT) + mirror = RMT::Mirror.new(logger: logger, mirror_src: RMT::Config.mirror_src_files?) + + ids = clean_target_input(ids) + raise RMT::CLI::Error.new(_('No repository IDs supplied')) if ids.empty? + + repos = [] + ids.each do |id| + repo = Repository.find_by!(scc_id: id) + raise RMT::CLI::Error.new(_('Mirroring of repository with ID %{repo_id} is not enabled') % { repo_id: id }) unless repo.mirroring_enabled + repos << repo + rescue ActiveRecord::RecordNotFound + raise RMT::CLI::Error.new(_('Repository with ID %{repo_id} not found') % { repo_id: id }) + end + + repos.each do |repo| + mirror_repo!(mirror, repo) + end + end + end + + desc 'product IDS', _('Mirror enabled repositories for a product with given product IDs') + def product(*targets) + RMT::Lockfile.lock do + logger = RMT::Logger.new(STDOUT) + mirror = RMT::Mirror.new(logger: logger, mirror_src: RMT::Config.mirror_src_files?) + + targets = clean_target_input(targets) + raise RMT::CLI::Error.new(_('No product IDs supplied')) if targets.empty? + + repos = [] + targets.each do |target| + products = Product.get_by_target!(target) + raise RMT::CLI::Error.new(_('Product for target %{target} not found') % { target: target }) if products.empty? + products.each do |product| + product_repos = product.repositories.where(mirroring_enabled: true) + raise RMT::CLI::Error.new(_('Product %{target} has no repositories enabled') % { target: target }) if product_repos.empty? + repos += product_repos.to_a + end + rescue ActiveRecord::RecordNotFound + raise RMT::CLI::Error.new(_('Product with ID %{target} not found') % { target: target }) + end + + repos.each do |repo| + mirror_repo!(mirror, repo) + end + end + end + + protected + + def mirror_repo!(mirror, repo) + mirror.mirror( + repository_url: repo.external_url, + local_path: Repository.make_local_path(repo.external_url), + auth_token: repo.auth_token, + repo_name: repo.name + ) + + repo.refresh_timestamp! + end + + +end diff --git a/lib/rmt/cli/products.rb b/lib/rmt/cli/products.rb index c783e523e..3b3307212 100644 --- a/lib/rmt/cli/products.rb +++ b/lib/rmt/cli/products.rb @@ -144,22 +144,9 @@ def change_product(target, set_enabled, all_modules) private def find_products(target) - product_id = Integer(target, 10) rescue nil - - products = [] - if product_id - product = Product.find(product_id) - products << product unless product.nil? - else - identifier, version, arch = target.split('/') - conditions = { identifier: identifier, version: version } - conditions[:arch] = arch if arch - products = Product.where(conditions).to_a - end - - products + Product.get_by_target!(target) rescue ActiveRecord::RecordNotFound - raise ProductNotFoundException.new(_('Product by ID %{id} not found.') % { id: product_id }) + raise ProductNotFoundException.new(_('Product by ID %{id} not found.') % { id: target }) end def repository_service diff --git a/spec/lib/rmt/cli/main_spec.rb b/spec/lib/rmt/cli/main_spec.rb index 27b6df124..a77c6a120 100644 --- a/spec/lib/rmt/cli/main_spec.rb +++ b/spec/lib/rmt/cli/main_spec.rb @@ -21,118 +21,6 @@ end end - describe 'mirror' do - let(:argv) { ['mirror'] } - - include_examples 'handles lockfile exception' - - context 'suma product tree mirror with exception' do - before do - create :repository, :with_products, mirroring_enabled: true - end - - it 'outputs exception message' do - expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree).and_raise(RMT::Mirror::Exception, 'black mirror') - expect_any_instance_of(RMT::Mirror).to receive(:mirror) - expect_any_instance_of(RMT::Logger).to receive(:warn).with('black mirror') - command - end - end - - context 'without repositories marked for mirroring' do - before do - create :repository, :with_products, mirroring_enabled: false - end - - it 'outputs a warning' do - expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) - expect_any_instance_of(RMT::Mirror).not_to receive(:mirror) - expect { command }.to raise_error(SystemExit).and output("There are no repositories marked for mirroring.\n").to_stderr.and output('').to_stdout - end - end - - context 'with repositories marked for mirroring' do - let!(:repository) { create :repository, :with_products, mirroring_enabled: true } - - it 'updates repository mirroring timestamp' do - expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) - expect_any_instance_of(RMT::Mirror).to receive(:mirror) - - Timecop.freeze(Time.utc(2018)) do - expect { command }.to change { repository.reload.last_mirrored_at }.to(DateTime.now.utc) - end - end - - context 'with exceptions during mirroring' do - before { allow_any_instance_of(RMT::Mirror).to receive(:mirror).and_raise(RMT::Mirror::Exception, 'black mirror') } - - it 'outputs exception message' do - expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) - expect_any_instance_of(RMT::Logger).to receive(:warn).with('black mirror') - command - end - end - end - - context 'with repositories changing during mirroring' do - let!(:repository) { create :repository, :with_products, mirroring_enabled: true } - let!(:additional_repository) { create :repository, :with_products, mirroring_enabled: false } - - it 'mirrors additional repositories' do - expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) - expect_any_instance_of(RMT::Mirror).to receive(:mirror).with( - repository_url: repository.external_url, - local_path: anything, - repo_name: anything, - auth_token: anything - ) do - # enable mirroring of the additional repository during mirroring - additional_repository.mirroring_enabled = true - additional_repository.save! - end - - expect_any_instance_of(RMT::Mirror).to receive(:mirror).with( - repository_url: additional_repository.external_url, - local_path: anything, - repo_name: anything, - auth_token: anything - ) - - command - end - end - - context 'with repositories changing during mirroring and exceptions occur' do - let!(:repository) { create :repository, :with_products, mirroring_enabled: true } - let!(:additional_repository) { create :repository, :with_products, mirroring_enabled: false } - - it 'handles exceptions and mirrors additional repositories' do - expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) - expect_any_instance_of(RMT::Mirror).to receive(:mirror).with( - repository_url: repository.external_url, - local_path: anything, - repo_name: anything, - auth_token: anything - ) do - # enable mirroring of the additional repository during mirroring - additional_repository.mirroring_enabled = true - additional_repository.save! - raise(RMT::Mirror::Exception, 'black mirror') - end - - expect_any_instance_of(RMT::Logger).to receive(:warn).with('black mirror') - expect_any_instance_of(RMT::Mirror).to receive(:mirror).with( - repository_url: additional_repository.external_url, - local_path: anything, - repo_name: anything, - auth_token: anything - ) - - command - end - end - end - describe 'help' do let(:argv) { ['help'] } diff --git a/spec/lib/rmt/cli/mirror_spec.rb b/spec/lib/rmt/cli/mirror_spec.rb new file mode 100644 index 000000000..75083107e --- /dev/null +++ b/spec/lib/rmt/cli/mirror_spec.rb @@ -0,0 +1,258 @@ +require 'rails_helper' + +RSpec.describe RMT::CLI::Mirror do + subject(:command) { described_class.start(argv) } + + let(:argv) { [] } + + describe 'mirror' do + let(:argv) { ['all'] } + + context 'lockfiles', :with_fakefs do + include_examples 'handles lockfile exception' + end + + context 'suma product tree mirror with exception' do + before do + create :repository, :with_products, mirroring_enabled: true + end + + it 'outputs exception message' do + expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree).and_raise(RMT::Mirror::Exception, 'black mirror') + expect_any_instance_of(RMT::Mirror).to receive(:mirror) + expect_any_instance_of(RMT::Logger).to receive(:warn).with('black mirror') + command + end + end + + context 'without repositories marked for mirroring' do + before do + create :repository, :with_products, mirroring_enabled: false + end + + it 'outputs a warning' do + expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) + expect_any_instance_of(RMT::Mirror).not_to receive(:mirror) + expect { command }.to raise_error(SystemExit).and output("There are no repositories marked for mirroring.\n").to_stderr.and output('').to_stdout + end + end + + context 'with repositories marked for mirroring' do + let!(:repository) { create :repository, :with_products, mirroring_enabled: true } + + it 'updates repository mirroring timestamp' do + expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) + expect_any_instance_of(RMT::Mirror).to receive(:mirror) + + Timecop.freeze(Time.utc(2018)) do + expect { command }.to change { repository.reload.last_mirrored_at }.to(Time.now.utc) + end + end + + context 'with exceptions during mirroring' do + before { allow_any_instance_of(RMT::Mirror).to receive(:mirror).and_raise(RMT::Mirror::Exception, 'black mirror') } + + it 'outputs exception message' do + expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) + expect_any_instance_of(RMT::Logger).to receive(:warn).with('black mirror') + command + end + end + end + + context 'with repositories changing during mirroring' do + let!(:repository) { create :repository, :with_products, mirroring_enabled: true } + let!(:additional_repository) { create :repository, :with_products, mirroring_enabled: false } + + it 'mirrors additional repositories' do + expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) + expect_any_instance_of(RMT::Mirror).to receive(:mirror).with( + repository_url: repository.external_url, + local_path: anything, + repo_name: anything, + auth_token: anything + ) do + # enable mirroring of the additional repository during mirroring + additional_repository.mirroring_enabled = true + additional_repository.save! + end + + expect_any_instance_of(RMT::Mirror).to receive(:mirror).with( + repository_url: additional_repository.external_url, + local_path: anything, + repo_name: anything, + auth_token: anything + ) + + command + end + end + + context 'with repositories changing during mirroring and exceptions occur' do + let!(:repository) { create :repository, :with_products, mirroring_enabled: true } + let!(:additional_repository) { create :repository, :with_products, mirroring_enabled: false } + + it 'handles exceptions and mirrors additional repositories' do + expect_any_instance_of(RMT::Mirror).to receive(:mirror_suma_product_tree) + expect_any_instance_of(RMT::Mirror).to receive(:mirror).with( + repository_url: repository.external_url, + local_path: anything, + repo_name: anything, + auth_token: anything + ) do + # enable mirroring of the additional repository during mirroring + additional_repository.mirroring_enabled = true + additional_repository.save! + raise(RMT::Mirror::Exception, 'black mirror') + end + + expect_any_instance_of(RMT::Logger).to receive(:warn).with('black mirror') + expect_any_instance_of(RMT::Mirror).to receive(:mirror).with( + repository_url: additional_repository.external_url, + local_path: anything, + repo_name: anything, + auth_token: anything + ) + + command + end + end + end + + describe 'mirror repository' do + context 'lockfiles', :with_fakefs do + include_examples 'handles lockfile exception' + end + + context 'when repository mirroring is enabled' do + let!(:repository) { create :repository, :with_products, mirroring_enabled: true } + let(:argv) { ['repository', repository.scc_id] } + + it 'mirrors the repository' do + expect_any_instance_of(RMT::Mirror).to receive(:mirror).with( + repository_url: repository.external_url, + local_path: anything, + repo_name: anything, + auth_token: anything + ) + + command + end + end + + context 'when repository mirroring is disabled' do + let!(:repository) { create :repository, :with_products, mirroring_enabled: false } + let(:argv) { ['repository', repository.scc_id] } + + it 'raises an error' do + expect { command }.to raise_error(SystemExit).and \ + output("Mirroring of repository with ID #{repository.scc_id} is not enabled\n").to_stderr.and \ + output('').to_stdout + end + end + + context 'when no repository IDs given' do + let(:argv) { ['repository'] } + + it 'raises an error' do + expect { command }.to raise_error(SystemExit).and \ + output("No repository IDs supplied\n").to_stderr.and \ + output('').to_stdout + end + end + + context 'when repository with given ID is not found' do + let(:argv) { ['repository', -42] } + + it 'raises an error' do + expect { command }.to raise_error(SystemExit).and \ + output("Repository with ID -42 not found\n").to_stderr.and \ + output('').to_stdout + end + end + end + + describe 'mirror product' do + context 'lockfiles', :with_fakefs do + include_examples 'handles lockfile exception' + end + + context 'when given an ID and product has enabled repos' do + let(:product) { create :product, :with_mirrored_repositories } + let(:argv) { ['product', product.id] } + + it 'mirrors repos' do + product.repositories.each do |repo| + expect_any_instance_of(RMT::Mirror).to receive(:mirror).with( + repository_url: repo.external_url, + local_path: anything, + repo_name: anything, + auth_token: anything + ) + end + + command + end + end + + context 'when given a triplet and product has enabled repos' do + let(:product) { create :product, :with_mirrored_repositories } + let(:argv) { ['product', [product.identifier, product.version, product.arch].join('/')] } + + it 'mirrors repos' do + product.repositories.each do |repo| + expect_any_instance_of(RMT::Mirror).to receive(:mirror).with( + repository_url: repo.external_url, + local_path: anything, + repo_name: anything, + auth_token: anything + ) + end + + command + end + end + + context 'when product has no mirrored repos' do + let(:product) { create :product, :with_not_mirrored_repositories } + let(:argv) { ['product', product.id] } + + it 'raises an error' do + expect { command }.to raise_error(SystemExit).and \ + output("Product #{product.id} has no repositories enabled\n").to_stderr.and \ + output('').to_stdout + end + end + + context 'when no product IDs given' do + let(:argv) { ['product'] } + + it 'raises an error' do + expect { command }.to raise_error(SystemExit).and \ + output("No product IDs supplied\n").to_stderr.and \ + output('').to_stdout + end + end + + context "when product with given ID doesn't exist" do + let(:argv) { ['product', 0] } + + it 'raises an error' do + expect { command }.to raise_error(SystemExit).and \ + output("Product with ID 0 not found\n").to_stderr.and \ + output('').to_stdout + end + end + + context "when product with given target doesn't exist" do + let(:target) { 'dummy/dummy/dummy' } + let(:argv) { ['product', target] } + + it 'raises an error' do + expect { command }.to raise_error(SystemExit).and \ + output("Product for target #{target} not found\n").to_stderr.and \ + output('').to_stdout + end + end + end +end From 64f5d3983fd14f3cbd721b98c096653e706da5ba Mon Sep 17 00:00:00 2001 From: Ivan Kapelyukhin Date: Tue, 13 Aug 2019 14:06:46 +0200 Subject: [PATCH 10/11] Bump the version; add an entry to changes file --- lib/rmt.rb | 2 +- package/obs/rmt-server.changes | 6 ++++++ package/obs/rmt-server.spec | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/rmt.rb b/lib/rmt.rb index 1bdf1dae8..5d0aa62dd 100644 --- a/lib/rmt.rb +++ b/lib/rmt.rb @@ -1,5 +1,5 @@ module RMT - VERSION ||= '2.3.7'.freeze + VERSION ||= '2.4.0'.freeze DEFAULT_USER = '_rmt'.freeze DEFAULT_GROUP = 'nginx'.freeze diff --git a/package/obs/rmt-server.changes b/package/obs/rmt-server.changes index 181b3d58a..da552261d 100644 --- a/package/obs/rmt-server.changes +++ b/package/obs/rmt-server.changes @@ -1,3 +1,9 @@ +------------------------------------------------------------------- +Tue Aug 20 11:12:58 UTC 2019 - Ivan Kapelyukhin + +- Version 2.4.0 +- Allow to mirror individual products/repositories (bsc#1138863) + ------------------------------------------------------------------- Fri Aug 16 11:07:26 UTC 2019 - tmuntaner@suse.com diff --git a/package/obs/rmt-server.spec b/package/obs/rmt-server.spec index 7a5121fbc..79bff5249 100644 --- a/package/obs/rmt-server.spec +++ b/package/obs/rmt-server.spec @@ -25,7 +25,7 @@ %define ruby_version %{rb_default_ruby_suffix} Name: rmt-server -Version: 2.3.7 +Version: 2.4.0 Release: 0 Summary: Repository mirroring tool and registration proxy for SCC License: GPL-2.0-or-later From 14474f21e2b31189af870a8357436ee856a5f5f9 Mon Sep 17 00:00:00 2001 From: Ivan Kapelyukhin Date: Tue, 20 Aug 2019 13:32:53 +0200 Subject: [PATCH 11/11] Handle mirroring exceptions when mirroring individual products/repos --- lib/rmt/cli/mirror.rb | 4 ++++ spec/lib/rmt/cli/mirror_spec.rb | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/lib/rmt/cli/mirror.rb b/lib/rmt/cli/mirror.rb index 331476d51..09a65f86d 100644 --- a/lib/rmt/cli/mirror.rb +++ b/lib/rmt/cli/mirror.rb @@ -50,6 +50,8 @@ def repository(*ids) repos.each do |repo| mirror_repo!(mirror, repo) + rescue RMT::Mirror::Exception => e + logger.warn e.to_s end end end @@ -78,6 +80,8 @@ def product(*targets) repos.each do |repo| mirror_repo!(mirror, repo) + rescue RMT::Mirror::Exception => e + logger.warn e.to_s end end end diff --git a/spec/lib/rmt/cli/mirror_spec.rb b/spec/lib/rmt/cli/mirror_spec.rb index 75083107e..9899efcac 100644 --- a/spec/lib/rmt/cli/mirror_spec.rb +++ b/spec/lib/rmt/cli/mirror_spec.rb @@ -140,6 +140,17 @@ end end + context 'when an exception is raised during mirroring' do + let!(:repository) { create :repository, :with_products, mirroring_enabled: true } + let(:argv) { ['repository', repository.scc_id] } + + it 'handles the exception and outputs a warning' do + expect_any_instance_of(RMT::Mirror).to receive(:mirror).at_least(:once).and_raise(RMT::Mirror::Exception, 'Dummy') + expect_any_instance_of(RMT::Logger).to receive(:warn).at_least(:once).with('Dummy') + command + end + end + context 'when repository mirroring is disabled' do let!(:repository) { create :repository, :with_products, mirroring_enabled: false } let(:argv) { ['repository', repository.scc_id] } @@ -213,6 +224,17 @@ end end + context 'when an exception is raised during mirroring' do + let(:product) { create :product, :with_mirrored_repositories } + let(:argv) { ['product', [product.identifier, product.version, product.arch].join('/')] } + + it 'handles the exception and outputs a warning' do + expect_any_instance_of(RMT::Mirror).to receive(:mirror).at_least(:once).and_raise(RMT::Mirror::Exception, 'Dummy') + expect_any_instance_of(RMT::Logger).to receive(:warn).at_least(:once).with('Dummy') + command + end + end + context 'when product has no mirrored repos' do let(:product) { create :product, :with_not_mirrored_repositories } let(:argv) { ['product', product.id] }